From 582be49f59151b9bffab7895306d8bb794fc0304 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 3 Mar 2020 22:47:18 +0000 Subject: [PATCH 01/40] feat: store pins in datastore instead of a DAG Adds a `.pins` datastore to `ipfs-repo` and uses that to store pins as cbor binary keyed by base64 stringified multihashes (n.b. not CIDs). Each pin has several fields: ```javascript { cid: // buffer, the full CID pinned type: // string, 'recursive' or 'direct' comments: // string, human-readable comments for the pin } ``` BREAKING CHANGES: * pins are now stored in a datastore, a repo migration will be necessary * ipfs.pins.add now returns an async generator * ipfs.pins.rm now returns an async generator Depends on: - [ ] https://github.com/ipfs/js-ipfs-repo/pull/221 --- examples/custom-ipfs-repo/package.json | 2 +- packages/interface-ipfs-core/SPEC/PIN.md | 21 +- packages/interface-ipfs-core/package.json | 1 + packages/interface-ipfs-core/src/block/rm.js | 11 +- packages/interface-ipfs-core/src/pin/add.js | 4 +- packages/interface-ipfs-core/src/pin/index.js | 3 +- packages/interface-ipfs-core/src/pin/ls.js | 52 +- packages/interface-ipfs-core/src/pin/pins.js | 460 ++++++++++++++++++ packages/interface-ipfs-core/src/pin/rm.js | 21 +- packages/interface-ipfs-core/src/pin/utils.js | 11 +- packages/interface-ipfs-core/src/repo/gc.js | 31 +- packages/ipfs-http-client/src/pin/add.js | 4 +- packages/ipfs-http-client/src/pin/rm.js | 7 +- packages/ipfs-mfs/package.json | 4 +- packages/ipfs/package.json | 4 +- packages/ipfs/src/cli/commands/pin/add.js | 6 +- packages/ipfs/src/cli/commands/pin/rm.js | 6 +- .../ipfs/src/core/components/add/index.js | 5 +- packages/ipfs/src/core/components/dag/put.js | 5 +- packages/ipfs/src/core/components/init.js | 1 - packages/ipfs/src/core/components/pin/add.js | 57 +-- packages/ipfs/src/core/components/pin/ls.js | 96 ++-- .../src/core/components/pin/pin-manager.js | 352 ++++++-------- .../ipfs/src/core/components/pin/pin-set.js | 317 ------------ .../ipfs/src/core/components/pin/pin.proto.js | 19 - packages/ipfs/src/core/components/pin/rm.js | 71 ++- packages/ipfs/src/core/components/repo/gc.js | 13 +- packages/ipfs/src/core/components/start.js | 2 +- packages/ipfs/src/http/api/resources/dag.js | 3 +- packages/ipfs/src/http/api/resources/pin.js | 5 +- packages/ipfs/test/cli/pin.js | 16 +- packages/ipfs/test/core/gc.spec.js | 13 +- packages/ipfs/test/core/node.js | 2 - packages/ipfs/test/core/pin-set.js | 188 ------- packages/ipfs/test/core/pin.js | 437 ----------------- packages/ipfs/test/core/pin.spec.js | 34 -- packages/ipfs/test/http-api/inject/pin.js | 9 +- 37 files changed, 855 insertions(+), 1438 deletions(-) create mode 100644 packages/interface-ipfs-core/src/pin/pins.js delete mode 100644 packages/ipfs/src/core/components/pin/pin-set.js delete mode 100644 packages/ipfs/src/core/components/pin/pin.proto.js delete mode 100644 packages/ipfs/test/core/pin-set.js delete mode 100644 packages/ipfs/test/core/pin.js delete mode 100644 packages/ipfs/test/core/pin.spec.js diff --git a/examples/custom-ipfs-repo/package.json b/examples/custom-ipfs-repo/package.json index 49fa3f558d..909d26dc5c 100644 --- a/examples/custom-ipfs-repo/package.json +++ b/examples/custom-ipfs-repo/package.json @@ -12,7 +12,7 @@ "dependencies": { "datastore-fs": "^0.9.1", "ipfs": "^0.41.0", - "ipfs-repo": "^0.30.1", + "ipfs-repo": "github:ipfs/js-ipfs-repo#store-pins-in-datastore", "it-all": "^1.0.1" }, "devDependencies": { diff --git a/packages/interface-ipfs-core/SPEC/PIN.md b/packages/interface-ipfs-core/SPEC/PIN.md index 0958ce80b7..1de696b088 100644 --- a/packages/interface-ipfs-core/SPEC/PIN.md +++ b/packages/interface-ipfs-core/SPEC/PIN.md @@ -8,11 +8,11 @@ > Adds an IPFS object to the pinset and also stores it to the IPFS repo. pinset is the set of hashes currently pinned (not gc'able). -##### `ipfs.pin.add(hash, [options])` +##### `ipfs.pin.add(source, [options])` Where: -- `hash` is an IPFS multihash. +- `source` is a [CID], an array of CIDs or an (async) iterable that yields CIDs - `options` is an object that can contain the following keys - `recursive` (`boolean`) - Recursively pin the object linked. Type: bool. Default: `true` - `timeout` (`number`|`string`) - Throw an error if the request does not complete within the specified milliseconds timeout. If `timeout` is a string, the value is parsed as a [human readable duration](https://www.npmjs.com/package/parse-duration). There is no timeout by default. @@ -21,9 +21,9 @@ Where: | Type | Description | | -------- | -------- | -| `Promise<{ cid: CID }>` | An array of objects that represent the files that were pinned | +| `AsyncIterable` | An async iterable that yields objects containing the CIDs that were pinned | -an array of objects is returned, each of the form: +Each yielded object has the form: ```JavaScript { @@ -77,10 +77,10 @@ A great source of [examples][] can be found in the tests for this API. > Remove a hash from the pinset -##### `ipfs.pin.rm(hash, [options])` +##### `ipfs.pin.rm(source, [options])` Where: -- `hash` is a multihash. +- `source` is a [CID], an array of CIDs or an (async) iterable that yields CIDs - `options` is an object that can contain the following keys - 'recursive' - Recursively unpin the object linked. Type: bool. Default: `true` @@ -88,15 +88,16 @@ Where: | Type | Description | | -------- | -------- | -| `Promise<{ cid: CID }>` | An array of unpinned objects | +| `AsyncIterable<{ cid: CID }>` | An async iterable that yields objects containing the CIDs that were unpinned | **Example:** ```JavaScript -const pinset = await ipfs.pin.rm('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') -console.log(pinset) +for await (const unpinned of ipfs.pin.rm(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) { + console.log(unpinned) +} // prints the hashes that were unpinned -// [ { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') } ] +// { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') } ``` A great source of [examples][] can be found in the tests for this API. diff --git a/packages/interface-ipfs-core/package.json b/packages/interface-ipfs-core/package.json index eb111e5028..78ce77471d 100644 --- a/packages/interface-ipfs-core/package.json +++ b/packages/interface-ipfs-core/package.json @@ -45,6 +45,7 @@ "is-ipfs": "^0.6.1", "it-all": "^1.0.1", "it-concat": "^1.0.0", + "it-drain": "^1.0.0", "it-last": "^1.0.1", "it-pushable": "^1.3.1", "multiaddr": "^7.2.1", diff --git a/packages/interface-ipfs-core/src/block/rm.js b/packages/interface-ipfs-core/src/block/rm.js index 37cd17d98f..ee43bdf203 100644 --- a/packages/interface-ipfs-core/src/block/rm.js +++ b/packages/interface-ipfs-core/src/block/rm.js @@ -4,6 +4,8 @@ const { getDescribe, getIt, expect } = require('../utils/mocha') const hat = require('hat') const all = require('it-all') +const last = require('it-last') +const drain = require('it-drain') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -142,13 +144,12 @@ module.exports = (common, options) => { format: 'raw', hashAlg: 'sha2-256' }) - await ipfs.pin.add(cid.toString()) + await drain(ipfs.pin.add(cid)) - const result = await all(ipfs.block.rm(cid)) + const result = await last(ipfs.block.rm(cid)) - expect(result).to.be.an('array').and.to.have.lengthOf(1) - expect(result[0]).to.have.property('error') - expect(result[0].error.message).to.include('pinned') + expect(result).to.have.property('error').that.is.an('Error') + .with.property('message').that.includes('pinned') }) }) } diff --git a/packages/interface-ipfs-core/src/pin/add.js b/packages/interface-ipfs-core/src/pin/add.js index 5e4cb2f3db..6c2e3559a4 100644 --- a/packages/interface-ipfs-core/src/pin/add.js +++ b/packages/interface-ipfs-core/src/pin/add.js @@ -28,8 +28,8 @@ module.exports = (common, options) => { after(() => common.clean()) it('should add a pin', async () => { - const pinset = await ipfs.pin.add(fixtures.files[0].cid, { recursive: false }) - expect(pinset.map(p => p.cid.toString())).to.include(fixtures.files[0].cid) + const pinset = await all(ipfs.pin.add(fixtures.files[0].cid, { recursive: false })) + expect(pinset.map(p => p.cid)).to.deep.include(fixtures.files[0].cid) }) }) } diff --git a/packages/interface-ipfs-core/src/pin/index.js b/packages/interface-ipfs-core/src/pin/index.js index b3723d07c5..ad97a96df0 100644 --- a/packages/interface-ipfs-core/src/pin/index.js +++ b/packages/interface-ipfs-core/src/pin/index.js @@ -4,7 +4,8 @@ const { createSuite } = require('../utils/suite') const tests = { ls: require('./ls'), rm: require('./rm'), - add: require('./add') + add: require('./add'), + core: require('./pins') } module.exports = createSuite(tests) diff --git a/packages/interface-ipfs-core/src/pin/ls.js b/packages/interface-ipfs-core/src/pin/ls.js index ffdfe7986b..29d21f2e9a 100644 --- a/packages/interface-ipfs-core/src/pin/ls.js +++ b/packages/interface-ipfs-core/src/pin/ls.js @@ -4,6 +4,7 @@ const { fixtures } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const all = require('it-all') +const drain = require('it-drain') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -23,22 +24,21 @@ module.exports = (common, options) => { ipfs = (await common.spawn()).api // two files wrapped in directories, only root CID pinned recursively const dir = fixtures.directory.files.map((file) => ({ path: file.path, content: file.data })) - await all(ipfs.add(dir, { pin: false, cidVersion: 0 })) - await ipfs.pin.add(fixtures.directory.cid, { recursive: true }) + await drain(ipfs.add(dir, { pin: false, cidVersion: 0 })) + await drain(ipfs.pin.add(fixtures.directory.cid, { recursive: true })) // a file (CID pinned recursively) - await all(ipfs.add(fixtures.files[0].data, { pin: false, cidVersion: 0 })) - await ipfs.pin.add(fixtures.files[0].cid, { recursive: true }) + await drain(ipfs.add(fixtures.files[0].data, { pin: false, cidVersion: 0 })) + await drain(ipfs.pin.add(fixtures.files[0].cid, { recursive: true })) // a single CID (pinned directly) - await all(ipfs.add(fixtures.files[1].data, { pin: false, cidVersion: 0 })) - await ipfs.pin.add(fixtures.files[1].cid, { recursive: false }) + await drain(ipfs.add(fixtures.files[1].data, { pin: false, cidVersion: 0 })) + await drain(ipfs.pin.add(fixtures.files[1].cid, { recursive: false })) }) after(() => common.clean()) // 1st, because ipfs.add pins automatically it('should list all recursive pins', async () => { - const pinset = (await all(ipfs.pin.ls({ type: 'recursive' }))) - .map(p => ({ ...p, cid: p.cid.toString() })) + const pinset = await all(ipfs.pin.ls({ type: 'recursive' })) expect(pinset).to.deep.include({ type: 'recursive', @@ -51,8 +51,7 @@ module.exports = (common, options) => { }) it('should list all indirect pins', async () => { - const pinset = (await all(ipfs.pin.ls({ type: 'indirect' }))) - .map(p => ({ ...p, cid: p.cid.toString() })) + const pinset = await all(ipfs.pin.ls({ type: 'indirect' })) expect(pinset).to.not.deep.include({ type: 'recursive', @@ -77,8 +76,7 @@ module.exports = (common, options) => { }) it('should list all types of pins', async () => { - const pinset = (await all(ipfs.pin.ls())) - .map(p => ({ ...p, cid: p.cid.toString() })) + const pinset = await all(ipfs.pin.ls()) expect(pinset).to.not.be.empty() // check the three "roots" @@ -107,15 +105,19 @@ module.exports = (common, options) => { it('should list all direct pins', async () => { const pinset = await all(ipfs.pin.ls({ type: 'direct' })) expect(pinset).to.have.lengthOf(1) - expect(pinset[0].type).to.equal('direct') - expect(pinset[0].cid.toString()).to.equal(fixtures.files[1].cid) + expect(pinset).to.deep.include({ + type: 'direct', + cid: fixtures.files[1].cid + }) }) it('should list pins for a specific hash', async () => { const pinset = await all(ipfs.pin.ls(fixtures.files[0].cid)) expect(pinset).to.have.lengthOf(1) - expect(pinset[0].type).to.equal('recursive') - expect(pinset[0].cid.toString()).to.equal(fixtures.files[0].cid) + expect(pinset).to.deep.include({ + type: 'recursive', + cid: fixtures.files[0].cid + }) }) it('should throw an error on missing direct pins for existing path', () => { @@ -136,22 +138,26 @@ module.exports = (common, options) => { it('should list indirect pins for a specific path', async () => { const pinset = await all(ipfs.pin.ls(`/ipfs/${fixtures.directory.cid}/files/ipfs.txt`, { type: 'indirect' })) expect(pinset).to.have.lengthOf(1) - expect(pinset[0].type).to.equal(`indirect through ${fixtures.directory.cid}`) - expect(pinset[0].cid.toString()).to.equal(fixtures.directory.files[1].cid) + expect(pinset).to.deep.include({ + type: `indirect through ${fixtures.directory.cid}`, + cid: fixtures.directory.files[1].cid + }) }) it('should list recursive pins for a specific hash', async () => { const pinset = await all(ipfs.pin.ls(fixtures.files[0].cid, { type: 'recursive' })) expect(pinset).to.have.lengthOf(1) - expect(pinset[0].type).to.equal('recursive') - expect(pinset[0].cid.toString()).to.equal(fixtures.files[0].cid) + expect(pinset).to.deep.include({ + type: 'recursive', + cid: fixtures.files[0].cid + }) }) it('should list pins for multiple CIDs', async () => { const pinset = await all(ipfs.pin.ls([fixtures.files[0].cid, fixtures.files[1].cid])) - const cids = pinset.map(p => p.cid.toString()) - expect(cids).to.include(fixtures.files[0].cid) - expect(cids).to.include(fixtures.files[1].cid) + const cids = pinset.map(p => p.cid) + expect(cids).to.deep.include(fixtures.files[0].cid) + expect(cids).to.deep.include(fixtures.files[1].cid) }) }) } diff --git a/packages/interface-ipfs-core/src/pin/pins.js b/packages/interface-ipfs-core/src/pin/pins.js new file mode 100644 index 0000000000..a171d0b91c --- /dev/null +++ b/packages/interface-ipfs-core/src/pin/pins.js @@ -0,0 +1,460 @@ +/* eslint-env mocha */ +'use strict' + +const { getDescribe, getIt, expect } = require('../utils/mocha') +const { + DAGNode +} = require('ipld-dag-pb') +const all = require('it-all') +const last = require('it-last') +const drain = require('it-drain') +const CID = require('cids') + +// fixture structure: +// planets/ +// solar-system.md +// mercury/ +// wiki.md +/* const pins = { + root: new CID('QmTAMavb995EHErSrKo7mB8dYkpaSJxu6ys1a6XJyB2sys'), + solarWiki: new CID('QmTMbkDfvHwq3Aup6Nxqn3KKw9YnoKzcZvuArAfQ9GF3QG'), + mercuryDir: new CID('QmbJCNKXJqVK8CzbjpNFz2YekHwh3CSHpBA86uqYg3sJ8q'), + mercuryWiki: new CID('QmVgSHAdMxFAuMP2JiMAYkB8pCWP1tcB9djqvq8GKAFiHi') +} */ +let pins +const pinTypes = { + direct: 'direct', + recursive: 'recursive', + indirect: 'indirect', + all: 'all' +} + +/** @typedef { import("ipfsd-ctl/src/factory") } Factory */ +/** + * @param {Factory} common + * @param {Object} options + */ +module.exports = (common, options) => { + const describe = getDescribe(options) + const it = getIt(options) + + describe('pin', function () { + this.timeout(50 * 1000) + + const fixtures = [{ + path: 'planets/mercury/wiki.md', + content: 'solar system content' + }, { + path: 'planets/solar-system.md', + content: 'wiki content' + }] + + let ipfs + + async function isPinnedWithType (path, type) { + try { + for await (const _ of ipfs.pin.ls(path, { type })) { // eslint-disable-line no-unused-vars + return true + } + return false + } catch (err) { + return false + } + } + + async function expectPinned (cid, type = pinTypes.all, pinned = true) { + if (typeof type === 'boolean') { + pinned = type + type = pinTypes.all + } + + const result = await isPinnedWithType(cid, type) + expect(result).to.eql(pinned) + } + + async function clearPins () { + for await (const { cid } of ipfs.pin.ls({ type: pinTypes.recursive })) { + await drain(ipfs.pin.rm(cid)) + } + + for await (const { cid } of ipfs.pin.ls({ type: pinTypes.direct })) { + await drain(ipfs.pin.rm(cid)) + } + } + + before(async () => { + ipfs = (await common.spawn()).api + const added = (await all(ipfs.add(fixtures))).reduce((acc, curr) => { + acc[curr.path] = curr.cid + + return acc + }, {}) + + pins = { + root: added.planets, + solarWiki: added['planets/solar-system.md'], + mercuryDir: added['planets/mercury'], + mercuryWiki: added['planets/mercury/wiki.md'] + } + }) + + after(() => common.clean()) + + describe('pinned status', function () { + beforeEach(async () => { + await clearPins() + await drain(ipfs.pin.add(pins.root)) + }) + + it('should be pinned when added', async () => { + await drain(ipfs.pin.add(pins.solarWiki)) + return expectPinned(pins.solarWiki) + }) + + it('should not be pinned when not in datastore', () => { + const falseHash = `${`${pins.root}`.slice(0, -2)}ss` + return expectPinned(falseHash, false) + }) + + it('should not be pinned when in datastore but not added', async () => { + await drain(ipfs.pin.rm(pins.root)) + return expectPinned(pins.root, false) + }) + + it('should be pinned recursively when added', () => { + return expectPinned(pins.root, pinTypes.recursive) + }) + + it('should be pinned indirectly', () => { + return expectPinned(pins.mercuryWiki, pinTypes.indirect) + }) + + it('should be pinned directly', async () => { + await drain(ipfs.pin.add(pins.mercuryDir, { recursive: false })) + return expectPinned(pins.mercuryDir, pinTypes.direct) + }) + + it('should not be pinned when not in datastore or added', async () => { + await clearPins() + return expectPinned(pins.mercuryDir, pinTypes.direct, false) + }) + }) + + describe('add', function () { + beforeEach(function () { + return clearPins() + }) + + it('should add recursively', async () => { + await drain(ipfs.pin.add(pins.root)) + await expectPinned(pins.root, pinTypes.recursive) + + const pinChecks = Object.values(pins).map(hash => expectPinned(hash)) + return Promise.all(pinChecks) + }) + + it('should add directly', async () => { + await drain(ipfs.pin.add(pins.root, { recursive: false })) + await Promise.all([ + expectPinned(pins.root, pinTypes.direct), + expectPinned(pins.solarWiki, false) + ]) + }) + + it('should recursively pin parent of direct pin', async () => { + await drain(ipfs.pin.add(pins.solarWiki, { recursive: false })) + await drain(ipfs.pin.add(pins.root)) + await Promise.all([ + // solarWiki is pinned both directly and indirectly o.O + expectPinned(pins.solarWiki, pinTypes.direct), + expectPinned(pins.solarWiki, pinTypes.indirect) + ]) + }) + + it('should fail to directly pin a recursive pin', async () => { + await drain(ipfs.pin.add(pins.root)) + return expect(last(ipfs.pin.add(pins.root, { recursive: false }))) + .to.eventually.be.rejected() + .with(/already pinned recursively/) + }) + + it('should fail to pin a hash not in datastore', function () { + this.timeout(5 * 1000) + const falseHash = `${`${pins.root}`.slice(0, -2)}ss` + return expect(last(ipfs.pin.add(falseHash, { timeout: '2s' }))) + .to.eventually.be.rejected() + // TODO: http api TimeoutErrors do not have this property + // .with.a.property('code').that.equals('ERR_TIMEOUT') + }) + + // TODO block rm breaks subsequent tests + // it.skip('needs all children in datastore to pin recursively', () => { + // return ipfs.block.rm(pins.mercuryWiki) + // .then(() => expectTimeout(pin.add(pins.root), 4000)) + // }) + }) + + describe('ls', function () { + before(async () => { + await clearPins() + await drain(ipfs.pin.add(pins.root)) + await drain(ipfs.pin.add(pins.mercuryDir, { recursive: false })) + }) + + it('should list pins of a particular CID', async () => { + const out = await all(ipfs.pin.ls(pins.mercuryDir)) + expect(out[0].cid).to.deep.equal(pins.mercuryDir) + expect(out[0].type).to.eql(pinTypes.direct) + }) + + it('should list indirect pins that supersede direct pins', async () => { + const ls = await all(ipfs.pin.ls()) + + const pinType = ls.find(out => out.cid.equals(pins.mercuryDir)).type + expect(pinType).to.eql(pinTypes.indirect) + }) + + it('should list all pins', async () => { + const out = await all(ipfs.pin.ls()) + + expect(out).to.deep.include.members([ + { + type: 'recursive', + cid: new CID(pins.root) + }, + { + type: 'indirect', + cid: new CID(pins.solarWiki) + }, + { + type: 'indirect', + cid: new CID(pins.mercuryDir) + }, + { + type: 'indirect', + cid: new CID(pins.mercuryWiki) + } + ]) + }) + + it('should list all direct pins', async () => { + const out = await all(ipfs.pin.ls({ type: 'direct' })) + + expect(out).to.deep.include.members([ + { + type: 'direct', + cid: new CID(pins.mercuryDir) + } + ]) + }) + + it('should list all recursive pins', async () => { + const out = await all(ipfs.pin.ls({ type: 'recursive' })) + + expect(out).to.deep.include.members([ + { + type: 'recursive', + cid: new CID(pins.root) + } + ]) + }) + + it('should list all indirect pins', async () => { + const out = await all(ipfs.pin.ls({ type: 'indirect' })) + + expect(out).to.deep.include.members([ + { + type: 'indirect', + cid: new CID(pins.solarWiki) + }, + { + type: 'indirect', + cid: new CID(pins.mercuryDir) + }, + { + type: 'indirect', + cid: new CID(pins.mercuryWiki) + } + ]) + }) + + it('should list direct pins for CID', async () => { + const out = await all(ipfs.pin.ls(pins.mercuryDir, { type: 'direct' })) + + expect(out).to.have.deep.members([ + { + type: 'direct', + cid: new CID(pins.mercuryDir) + } + ]) + }) + + it('should list direct pins for path', async () => { + const out = await all(ipfs.pin.ls(`/ipfs/${pins.root}/mercury/`, { type: 'direct' })) + + expect(out).to.have.deep.members([ + { + type: 'direct', + cid: new CID(pins.mercuryDir) + } + ]) + }) + + it('should list direct pins for path (no match)', () => { + return expect(all(ipfs.pin.ls(`/ipfs/${pins.root}/mercury/wiki.md`, { type: 'direct' }))) + .to.eventually.be.rejected() + }) + + it('should list direct pins for CID (no match)', () => { + return expect(all(ipfs.pin.ls(pins.root, { type: 'direct' }))) + .to.eventually.be.rejected() + }) + + it('should list recursive pins for CID', async () => { + const out = await all(ipfs.pin.ls(pins.root, { type: 'recursive' })) + + expect(out).to.have.deep.members([ + { + type: 'recursive', + cid: new CID(pins.root) + } + ]) + }) + + it('should list recursive pins for CID (no match)', () => { + return expect(all(ipfs.pin.ls(pins.mercuryDir, { type: 'recursive' }))) + .to.eventually.be.rejected() + }) + + it('should list indirect pins for CID', async () => { + const out = await all(ipfs.pin.ls(pins.solarWiki, { type: 'indirect' })) + + expect(out).to.have.deep.members([ + { + type: `indirect through ${pins.root}`, + cid: new CID(pins.solarWiki) + } + ]) + }) + + it('should list indirect pins for CID (no match)', () => { + return expect(all(ipfs.pin.ls(pins.root, { type: 'indirect' }))) + .to.eventually.be.rejected() + }) + }) + + describe('rm', function () { + beforeEach(async () => { + await clearPins() + await drain(ipfs.pin.add(pins.root)) + }) + + it('should remove a recursive pin', async () => { + await drain(ipfs.pin.rm(pins.root)) + await Promise.all([ + expectPinned(pins.root, false), + expectPinned(pins.mercuryWiki, false) + ]) + }) + + it('should remove a direct pin', async () => { + await clearPins() + await drain(ipfs.pin.add(pins.mercuryDir, { recursive: false })) + await drain(ipfs.pin.rm(pins.mercuryDir)) + await expectPinned(pins.mercuryDir, false) + }) + + it('should fail to remove an indirect pin', async () => { + await expect(last(ipfs.pin.rm(pins.solarWiki))) + .to.eventually.be.rejected() + .with(/is pinned indirectly under/) + await expectPinned(pins.solarWiki) + }) + + it('should fail when an item is not pinned', async () => { + await drain(ipfs.pin.rm(pins.root)) + await expect(last(ipfs.pin.rm(pins.root))) + .to.eventually.be.rejected() + .with(/is not pinned/) + }) + }) + + describe('non-dag-pb nodes', function () { + it('should pin dag-cbor', async () => { + const cid = await ipfs.dag.put({}, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + await drain(ipfs.pin.add(cid)) + + const pins = await all(ipfs.pin.ls()) + + expect(pins).to.deep.include({ + type: 'recursive', + cid + }) + }) + + it('should pin raw', async () => { + const cid = await ipfs.dag.put(Buffer.alloc(0), { + format: 'raw', + hashAlg: 'sha2-256' + }) + + await drain(ipfs.pin.add(cid)) + + const pins = await all(ipfs.pin.ls()) + + expect(pins).to.deep.include({ + type: 'recursive', + cid + }) + }) + + it('should pin dag-cbor with dag-pb child', async () => { + const child = await ipfs.dag.put(new DAGNode(Buffer.from(`${Math.random()}`)), { + format: 'dag-pb', + hashAlg: 'sha2-256' + }) + const parent = await ipfs.dag.put({ + child + }, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + await drain(ipfs.pin.add(parent, { + recursive: true + })) + + const pins = await all(ipfs.pin.ls()) + + expect(pins).to.deep.include({ + cid: parent, + type: 'recursive' + }) + expect(pins).to.deep.include({ + cid: child, + type: 'indirect' + }) + }) + }) + + describe('ls', () => { + it('should throw error for invalid non-string pin type option', () => { + return expect(all(ipfs.pin.ls({ type: 6 }))) + .to.eventually.be.rejected() + // TODO: go-ipfs does not return error codes + // .with.property('code').that.equals('ERR_INVALID_PIN_TYPE') + }) + + it('should throw error for invalid string pin type option', () => { + return expect(all(ipfs.pin.ls({ type: '__proto__' }))) + .to.eventually.be.rejected() + // TODO: go-ipfs does not return error codes + // .with.property('code').that.equals('ERR_INVALID_PIN_TYPE') + }) + }) + }) +} diff --git a/packages/interface-ipfs-core/src/pin/rm.js b/packages/interface-ipfs-core/src/pin/rm.js index 7565381687..91bc802ea5 100644 --- a/packages/interface-ipfs-core/src/pin/rm.js +++ b/packages/interface-ipfs-core/src/pin/rm.js @@ -4,6 +4,7 @@ const { fixtures } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const all = require('it-all') +const drain = require('it-drain') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -20,31 +21,31 @@ module.exports = (common, options) => { let ipfs before(async () => { ipfs = (await common.spawn()).api - await all(ipfs.add(fixtures.files[0].data, { pin: false })) - await ipfs.pin.add(fixtures.files[0].cid, { recursive: true }) - await all(ipfs.add(fixtures.files[1].data, { pin: false })) - await ipfs.pin.add(fixtures.files[1].cid, { recursive: false }) + await drain(ipfs.add(fixtures.files[0].data, { pin: false })) + await drain(ipfs.pin.add(fixtures.files[0].cid, { recursive: true })) + await drain(ipfs.add(fixtures.files[1].data, { pin: false })) + await drain(ipfs.pin.add(fixtures.files[1].cid, { recursive: false })) }) after(() => common.clean()) it('should remove a recursive pin', async () => { - const removedPinset = await ipfs.pin.rm(fixtures.files[0].cid, { recursive: true }) - expect(removedPinset.map(p => p.cid.toString())).to.deep.equal([fixtures.files[0].cid]) + const removedPinset = await all(ipfs.pin.rm(fixtures.files[0].cid, { recursive: true })) + expect(removedPinset.map(p => p.cid)).to.deep.equal([fixtures.files[0].cid]) const pinset = await all(ipfs.pin.ls({ type: 'recursive' })) - expect(pinset.map(p => ({ ...p, cid: p.cid.toString() }))).to.not.deep.include({ + expect(pinset).to.not.deep.include({ cid: fixtures.files[0].cid, type: 'recursive' }) }) it('should remove a direct pin', async () => { - const removedPinset = await ipfs.pin.rm(fixtures.files[1].cid, { recursive: false }) - expect(removedPinset.map(p => p.cid.toString())).to.deep.equal([fixtures.files[1].cid]) + const removedPinset = await all(ipfs.pin.rm(fixtures.files[1].cid, { recursive: false })) + expect(removedPinset.map(p => p.cid)).to.deep.equal([fixtures.files[1].cid]) const pinset = await all(ipfs.pin.ls({ type: 'direct' })) - expect(pinset.map(p => p.cid.toString())).to.not.include(fixtures.files[1].cid) + expect(pinset.map(p => p.cid)).to.not.deep.include(fixtures.files[1].cid) }) }) } diff --git a/packages/interface-ipfs-core/src/pin/utils.js b/packages/interface-ipfs-core/src/pin/utils.js index 3ee67f0d53..34b33eee6b 100644 --- a/packages/interface-ipfs-core/src/pin/utils.js +++ b/packages/interface-ipfs-core/src/pin/utils.js @@ -1,26 +1,27 @@ 'use strict' const loadFixture = require('aegir/fixtures') +const CID = require('cids') exports.fixtures = Object.freeze({ // NOTE: files under 'directory' need to be different than standalone ones in 'files' directory: Object.freeze({ - cid: 'QmY8KdYQSYKFU5hM7F5ioZ5yYSgV5VZ1kDEdqfRL3rFgcd', + cid: new CID('QmY8KdYQSYKFU5hM7F5ioZ5yYSgV5VZ1kDEdqfRL3rFgcd'), files: Object.freeze([Object.freeze({ path: 'test-folder/ipfs-add.js', data: loadFixture('test/fixtures/test-folder/ipfs-add.js', 'interface-ipfs-core'), - cid: 'QmbKtKBrmeRHjNCwR4zAfCJdMVu6dgmwk9M9AE9pUM9RgG' + cid: new CID('QmbKtKBrmeRHjNCwR4zAfCJdMVu6dgmwk9M9AE9pUM9RgG') }), Object.freeze({ path: 'test-folder/files/ipfs.txt', data: loadFixture('test/fixtures/test-folder/files/ipfs.txt', 'interface-ipfs-core'), - cid: 'QmdFyxZXsFiP4csgfM5uPu99AvFiKH62CSPDw5TP92nr7w' + cid: new CID('QmdFyxZXsFiP4csgfM5uPu99AvFiKH62CSPDw5TP92nr7w') })]) }), files: Object.freeze([Object.freeze({ data: loadFixture('test/fixtures/testfile.txt', 'interface-ipfs-core'), - cid: 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP' + cid: new CID('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') }), Object.freeze({ data: loadFixture('test/fixtures/test-folder/files/hello.txt', 'interface-ipfs-core'), - cid: 'QmY9cxiHqTFoWamkQVkpmmqzBrY3hCBEL2XNu3NtX74Fuu' + cid: new CID('QmY9cxiHqTFoWamkQVkpmmqzBrY3hCBEL2XNu3NtX74Fuu') })]) }) diff --git a/packages/interface-ipfs-core/src/repo/gc.js b/packages/interface-ipfs-core/src/repo/gc.js index da4966250a..dcf55652ab 100644 --- a/packages/interface-ipfs-core/src/repo/gc.js +++ b/packages/interface-ipfs-core/src/repo/gc.js @@ -4,6 +4,7 @@ const { getDescribe, getIt, expect } = require('../utils/mocha') const { DAGNode } = require('ipld-dag-pb') const all = require('it-all') +const drain = require('it-drain') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -29,8 +30,8 @@ module.exports = (common, options) => { const pinset = await all(ipfs.pin.ls()) expect(pinset.map(obj => obj.cid.toString())).includes(res[0].cid.toString()) - await ipfs.pin.rm(res[0].cid) - await all(ipfs.repo.gc()) + await drain(ipfs.pin.rm(res[0].cid)) + await drain(ipfs.repo.gc()) const finalPinset = await all(ipfs.pin.ls()) expect(finalPinset.map(obj => obj.cid.toString())).not.includes(res[0].cid.toString()) @@ -53,7 +54,7 @@ module.exports = (common, options) => { expect(refsAfterAdd.map(r => r.ref)).includes(cid.toString()) // Run garbage collection - await all(ipfs.repo.gc()) + await drain(ipfs.repo.gc()) // Get the list of local blocks after GC, should still contain the hash, // because the file is still pinned @@ -61,7 +62,7 @@ module.exports = (common, options) => { expect(refsAfterGc.map(r => r.ref)).includes(cid.toString()) // Unpin the data - await ipfs.pin.rm(cid) + await drain(ipfs.pin.rm(cid)) // Run garbage collection await all(ipfs.repo.gc()) @@ -88,7 +89,7 @@ module.exports = (common, options) => { expect(refsAfterAdd.map(r => r.ref)).includes(hash) // Run garbage collection - await all(ipfs.repo.gc()) + await drain(ipfs.repo.gc()) // Get the list of local blocks after GC, should still contain the hash, // because the file is in MFS @@ -99,7 +100,7 @@ module.exports = (common, options) => { await ipfs.files.rm('/test') // Run garbage collection - await all(ipfs.repo.gc()) + await drain(ipfs.repo.gc()) // The list of local blocks should no longer contain the hash const refsAfterUnpinAndGc = await all(ipfs.refs.local()) @@ -131,7 +132,7 @@ module.exports = (common, options) => { expect(hashesAfterAdd).includes(dataCid.toString()) // Run garbage collection - await all(ipfs.repo.gc()) + await drain(ipfs.repo.gc()) // Get the list of local blocks after GC, should still contain the hash, // because the file is pinned and in MFS @@ -143,7 +144,7 @@ module.exports = (common, options) => { await ipfs.files.rm('/test') // Run garbage collection - await all(ipfs.repo.gc()) + await drain(ipfs.repo.gc()) // Get the list of local blocks after GC, should still contain the hash, // because the file is still pinned @@ -153,10 +154,10 @@ module.exports = (common, options) => { expect(hashesAfterRmAndGc).includes(dataCid.toString()) // Unpin the data - await ipfs.pin.rm(dataCid) + await drain(ipfs.pin.rm(dataCid)) // Run garbage collection - await all(ipfs.repo.gc()) + await drain(ipfs.repo.gc()) // The list of local blocks should no longer contain the hashes const refsAfterUnpinAndGc = await all(ipfs.refs.local()) @@ -174,7 +175,7 @@ module.exports = (common, options) => { const dataCid = addRes[0].cid // Unpin the data - await ipfs.pin.rm(dataCid) + await drain(ipfs.pin.rm(dataCid)) // Create a link to the data from an object const obj = await new DAGNode(Buffer.from('fruit'), [{ @@ -198,14 +199,14 @@ module.exports = (common, options) => { expect(hashesAfterAdd).includes(dataCid.toString()) // Recursively pin the object - await ipfs.pin.add(objCid, { recursive: true }) + await drain(ipfs.pin.add(objCid, { recursive: true })) // The data should now be indirectly pinned const pins = await all(ipfs.pin.ls()) expect(pins.find(p => p.cid.toString() === dataCid.toString()).type).to.eql('indirect') // Run garbage collection - await all(ipfs.repo.gc()) + await drain(ipfs.repo.gc()) // Get the list of local blocks after GC, should still contain the data // hash, because the data is still (indirectly) pinned @@ -213,10 +214,10 @@ module.exports = (common, options) => { expect(refsAfterGc.map(r => r.ref)).includes(dataCid.toString()) // Recursively unpin the object - await ipfs.pin.rm(objCid.toString()) + await drain(ipfs.pin.rm(objCid.toString())) // Run garbage collection - await all(ipfs.repo.gc()) + await drain(ipfs.repo.gc()) // The list of local blocks should no longer contain the hashes const refsAfterUnpinAndGc = await all(ipfs.refs.local()) diff --git a/packages/ipfs-http-client/src/pin/add.js b/packages/ipfs-http-client/src/pin/add.js index 41b119fe87..b42cbb03fd 100644 --- a/packages/ipfs-http-client/src/pin/add.js +++ b/packages/ipfs-http-client/src/pin/add.js @@ -4,7 +4,7 @@ const CID = require('cids') const configure = require('../lib/configure') module.exports = configure(({ ky }) => { - return async (paths, options) => { + return async function * (paths, options) { paths = Array.isArray(paths) ? paths : [paths] options = options || {} @@ -19,6 +19,6 @@ module.exports = configure(({ ky }) => { searchParams }).json() - return (res.Pins || []).map(cid => ({ cid: new CID(cid) })) + yield * (res.Pins || []).map(cid => ({ cid: new CID(cid) })) } }) diff --git a/packages/ipfs-http-client/src/pin/rm.js b/packages/ipfs-http-client/src/pin/rm.js index 83fbca93c4..0a2d46a22b 100644 --- a/packages/ipfs-http-client/src/pin/rm.js +++ b/packages/ipfs-http-client/src/pin/rm.js @@ -4,11 +4,12 @@ const CID = require('cids') const configure = require('../lib/configure') module.exports = configure(({ ky }) => { - return async (path, options) => { + return async function * (paths, options) { + paths = Array.isArray(paths) ? paths : [paths] options = options || {} const searchParams = new URLSearchParams(options.searchParams) - searchParams.set('arg', `${path}`) + paths.forEach(path => searchParams.append('arg', `${path}`)) if (options.recursive != null) searchParams.set('recursive', options.recursive) const res = await ky.post('pin/rm', { @@ -18,6 +19,6 @@ module.exports = configure(({ ky }) => { searchParams }).json() - return (res.Pins || []).map(cid => ({ cid: new CID(cid) })) + yield * (res.Pins || []).map(cid => ({ cid: new CID(cid) })) } }) diff --git a/packages/ipfs-mfs/package.json b/packages/ipfs-mfs/package.json index 75023d1ee7..5d96d455a0 100644 --- a/packages/ipfs-mfs/package.json +++ b/packages/ipfs-mfs/package.json @@ -48,7 +48,7 @@ "form-data": "^3.0.0", "ipfs-block": "^0.8.1", "ipfs-block-service": "^0.16.0", - "ipfs-repo": "^0.30.1", + "ipfs-repo": "github:ipfs/js-ipfs-repo#store-pins-in-datastore", "ipld": "^0.25.0", "memdown": "^5.1.0", "nyc": "^15.0.0", @@ -73,8 +73,8 @@ "ipld-dag-pb": "^0.18.2", "it-all": "^1.0.1", "it-last": "^1.0.1", - "it-to-stream": "^0.1.1", "it-pipe": "^1.1.0", + "it-to-stream": "^0.1.1", "joi-browser": "^13.4.0", "mortice": "^2.0.0", "multicodec": "^1.0.0", diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 84e5f1e69a..333633a58b 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -76,6 +76,7 @@ "bl": "^4.0.0", "bs58": "^4.0.1", "byteman": "^1.3.5", + "cbor": "^5.0.1", "cid-tool": "^0.4.0", "cids": "^0.7.3", "class-is": "^1.1.0", @@ -99,7 +100,7 @@ "ipfs-http-response": "^0.5.0", "ipfs-mfs": "^1.0.0", "ipfs-multipart": "^0.3.0", - "ipfs-repo": "^0.30.1", + "ipfs-repo": "github:ipfs/js-ipfs-repo#store-pins-in-datastore", "ipfs-unixfs": "^1.0.0", "ipfs-unixfs-exporter": "^1.0.1", "ipfs-unixfs-importer": "^1.0.1", @@ -117,6 +118,7 @@ "is-ipfs": "^0.6.1", "it-all": "^1.0.1", "it-concat": "^1.0.0", + "it-drain": "^1.0.0", "it-glob": "0.0.7", "it-last": "^1.0.1", "it-pipe": "^1.1.0", diff --git a/packages/ipfs/src/cli/commands/pin/add.js b/packages/ipfs/src/cli/commands/pin/add.js index b4e07130dd..a1273e9c33 100644 --- a/packages/ipfs/src/cli/commands/pin/add.js +++ b/packages/ipfs/src/cli/commands/pin/add.js @@ -25,9 +25,9 @@ module.exports = { async handler ({ ctx, ipfsPath, recursive, cidBase }) { const { ipfs, print } = ctx const type = recursive ? 'recursive' : 'direct' - const results = await ipfs.pin.add(ipfsPath, { recursive }) - results.forEach((res) => { + + for await (const res of ipfs.pin.add(ipfsPath, { recursive })) { print(`pinned ${cidToString(res.cid, { base: cidBase })} ${type}ly`) - }) + } } } diff --git a/packages/ipfs/src/cli/commands/pin/rm.js b/packages/ipfs/src/cli/commands/pin/rm.js index bcf2c438f8..7ad2a4b640 100644 --- a/packages/ipfs/src/cli/commands/pin/rm.js +++ b/packages/ipfs/src/cli/commands/pin/rm.js @@ -24,9 +24,9 @@ module.exports = { async handler ({ ctx, ipfsPath, recursive, cidBase }) { const { ipfs, print } = ctx - const results = await ipfs.pin.rm(ipfsPath, { recursive }) - results.forEach((res) => { + + for await (const res of ipfs.pin.rm(ipfsPath, { recursive })) { print(`unpinned ${cidToString(res.cid, { base: cidBase })}`) - }) + } } } diff --git a/packages/ipfs/src/core/components/add/index.js b/packages/ipfs/src/core/components/add/index.js index 40e69a7fe0..1c5aab9e3e 100644 --- a/packages/ipfs/src/core/components/add/index.js +++ b/packages/ipfs/src/core/components/add/index.js @@ -4,6 +4,7 @@ const importer = require('ipfs-unixfs-importer') const normaliseAddInput = require('ipfs-utils/src/files/normalise-input') const { parseChunkerString } = require('./utils') const pipe = require('it-pipe') +const drain = require('it-drain') module.exports = ({ ipld, gcLock, preload, pin, options: constructorOptions }) => { const isShardingEnabled = constructorOptions.EXPERIMENTAL && constructorOptions.EXPERIMENTAL.sharding @@ -111,10 +112,10 @@ function pinFile (pin, opts) { if (shouldPin) { // Note: addAsyncIterator() has already taken a GC lock, so tell // pin.add() not to take a (second) GC lock - await pin.add(file.cid, { + await drain(pin.add(file.cid, { preload: false, lock: false - }) + })) } yield file diff --git a/packages/ipfs/src/core/components/dag/put.js b/packages/ipfs/src/core/components/dag/put.js index 301c87ba8c..e1e6955b0d 100644 --- a/packages/ipfs/src/core/components/dag/put.js +++ b/packages/ipfs/src/core/components/dag/put.js @@ -2,6 +2,7 @@ const multicodec = require('multicodec') const nameToCodec = name => multicodec[name.toUpperCase().replace(/-/g, '_')] +const drain = require('it-drain') module.exports = ({ ipld, pin, gcLock, preload }) => { return async function put (dagNode, options) { @@ -51,9 +52,9 @@ module.exports = ({ ipld, pin, gcLock, preload }) => { }) if (options.pin) { - await pin.add(cid, { + await drain(pin.add(cid, { lock: false - }) + })) } if (options.preload !== false) { diff --git a/packages/ipfs/src/core/components/init.js b/packages/ipfs/src/core/components/init.js index 7e2f0ce94d..2fda6c3490 100644 --- a/packages/ipfs/src/core/components/init.js +++ b/packages/ipfs/src/core/components/init.js @@ -110,7 +110,6 @@ module.exports = ({ } const pinManager = new PinManager(repo, dag) - await pinManager.load() const pin = { add: Components.pin.add({ pinManager, gcLock, dag }), diff --git a/packages/ipfs/src/core/components/pin/add.js b/packages/ipfs/src/core/components/pin/add.js index 8f4a35c223..8376e57b5f 100644 --- a/packages/ipfs/src/core/components/pin/add.js +++ b/packages/ipfs/src/core/components/pin/add.js @@ -2,57 +2,33 @@ 'use strict' const { resolvePath, withTimeoutOption } = require('../../utils') +const PinManager = require('./pin-manager') +const { PinTypes } = PinManager module.exports = ({ pinManager, gcLock, dag }) => { - return withTimeoutOption(async function add (paths, options) { + return withTimeoutOption(async function * add (paths, options) { options = options || {} const recursive = options.recursive !== false const cids = await resolvePath(dag, paths, { signal: options.signal }) - const pinAdd = async () => { - const results = [] - + const pinAdd = async function * () { // verify that each hash can be pinned for (const cid of cids) { - const key = cid.toBaseEncodedString() - - if (recursive) { - if (pinManager.recursivePins.has(key)) { - // it's already pinned recursively - results.push(cid) + const { reason } = await pinManager.isPinnedWithType(cid, [PinTypes.recursive, PinTypes.direct]) - continue - } - - // entire graph of nested links should be pinned, - // so make sure we have all the objects - await pinManager.fetchCompleteDag(key, { preload: options.preload, signal: options.signal }) + if (reason === 'recursive' && !recursive) { + // only disallow trying to override recursive pins + throw new Error(`${cid} already pinned recursively`) + } - // found all objects, we can add the pin - results.push(cid) + if (recursive) { + await pinManager.pinRecursively(cid) } else { - if (pinManager.recursivePins.has(key)) { - // recursive supersedes direct, can't have both - throw new Error(`${key} already pinned recursively`) - } - - if (!pinManager.directPins.has(key)) { - // make sure we have the object - await dag.get(cid, { preload: options.preload }) - } - - results.push(cid) + await pinManager.pinDirectly(cid) } - } - // update the pin sets in memory - const pinset = recursive ? pinManager.recursivePins : pinManager.directPins - results.forEach(cid => pinset.add(cid.toString())) - - // persist updated pin sets to datastore - await pinManager.flushPins() - - return results.map(cid => ({ cid })) + yield { cid } + } } // When adding a file, we take a lock that gets released after pinning @@ -60,13 +36,14 @@ module.exports = ({ pinManager, gcLock, dag }) => { const lock = Boolean(options.lock) if (!lock) { - return pinAdd() + yield * pinAdd() + return } const release = await gcLock.readLock() try { - await pinAdd() + yield * pinAdd() } finally { release() } diff --git a/packages/ipfs/src/core/components/pin/ls.js b/packages/ipfs/src/core/components/pin/ls.js index 253384c5fb..bfdff150e7 100644 --- a/packages/ipfs/src/core/components/pin/ls.js +++ b/packages/ipfs/src/core/components/pin/ls.js @@ -1,13 +1,22 @@ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { parallelMap } = require('streaming-iterables') -const CID = require('cids') const { resolvePath } = require('../../utils') const PinManager = require('./pin-manager') const { PinTypes } = PinManager -const PIN_LS_CONCURRENCY = 8 +function toPin (type, cid, name) { + const output = { + type, + cid + } + + if (name) { + output.name = name + } + + return output +} module.exports = ({ pinManager, dag }) => { return async function * ls (paths, options) { @@ -25,67 +34,70 @@ module.exports = ({ pinManager, dag }) => { if (typeof options.type === 'string') { type = options.type.toLowerCase() } - const err = PinManager.checkPinType(type) - if (err) { - throw err - } + + PinManager.checkPinType(type) + } else { + options.type = PinTypes.all } if (paths) { paths = Array.isArray(paths) ? paths : [paths] // check the pinned state of specific hashes - const cids = await resolvePath(dag, paths) + const cids = await resolvePath(dag, paths, { signal: options.signal }) + let noMatch = true - yield * parallelMap(PIN_LS_CONCURRENCY, async cid => { - const { reason, pinned } = await pinManager.isPinnedWithType(cid, type) + for (const cid of cids) { + const { reason, pinned, parent } = await pinManager.isPinnedWithType(cid, type) if (!pinned) { - throw new Error(`path '${paths[cids.indexOf(cid)]}' is not pinned`) + throw new Error(`path '${paths}' is not pinned`) } - if (reason === PinTypes.direct || reason === PinTypes.recursive) { - return { cid, type: reason } + switch (reason) { + case PinTypes.direct: + case PinTypes.recursive: + noMatch = false + yield { + type: reason, + cid + } + break + default: + noMatch = false + yield { + type: `${PinTypes.indirect} through ${parent}`, + cid + } } + } - return { cid, type: `${PinTypes.indirect} through ${reason}` } - }, cids) + if (noMatch) { + throw new Error('No match found') + } return } - // show all pinned items of type - let pins = [] - - if (type === PinTypes.direct || type === PinTypes.all) { - pins = pins.concat( - Array.from(pinManager.directPins).map(cid => ({ - type: PinTypes.direct, - cid: new CID(cid) - })) - ) - } - if (type === PinTypes.recursive || type === PinTypes.all) { - pins = pins.concat( - Array.from(pinManager.recursivePins).map(cid => ({ - type: PinTypes.recursive, - cid: new CID(cid) - })) - ) + for await (const { cid, name } of pinManager.recursiveKeys()) { + yield toPin(PinTypes.recursive, cid, name) + } } if (type === PinTypes.indirect || type === PinTypes.all) { - const indirects = await pinManager.getIndirectKeys(options) - - pins = pins - // if something is pinned both directly and indirectly, - // report the indirect entry - .filter(({ cid }) => !indirects.includes(cid.toString()) || !pinManager.directPins.has(cid.toString())) - .concat(indirects.map(cid => ({ type: PinTypes.indirect, cid: new CID(cid) }))) + for await (const cid of pinManager.indirectKeys(options)) { + yield { + type: PinTypes.indirect, + cid + } + } } - // FIXME: https://github.com/ipfs/js-ipfs/issues/2244 - yield * pins + if (type === PinTypes.direct || type === PinTypes.all) { + for await (const { cid, name } of pinManager.directKeys()) { + yield toPin(PinTypes.direct, cid, name) + } + } } } diff --git a/packages/ipfs/src/core/components/pin/pin-manager.js b/packages/ipfs/src/core/components/pin/pin-manager.js index 8cd8c1c261..24ea7c480d 100644 --- a/packages/ipfs/src/core/components/pin/pin-manager.js +++ b/packages/ipfs/src/core/components/pin/pin-manager.js @@ -1,31 +1,29 @@ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { DAGNode, DAGLink } = require('ipld-dag-pb') const CID = require('cids') -const { default: Queue } = require('p-queue') -const { Key } = require('interface-datastore') const errCode = require('err-code') -const multicodec = require('multicodec') const dagCborLinks = require('dag-cbor-links') const debug = require('debug') -const { cidToString } = require('../../../utils/cid') - -const createPinSet = require('./pin-set') - -const { Errors } = require('interface-datastore') -const ERR_NOT_FOUND = Errors.notFoundError().code +// const parallelBatch = require('it-parallel-batch') +const first = require('it-first') +const all = require('it-all') +const cbor = require('cbor') // arbitrary limit to the number of concurrent dag operations -const WALK_DAG_CONCURRENCY_LIMIT = 300 -const IS_PINNED_WITH_TYPE_CONCURRENCY_LIMIT = 300 -const PIN_DS_KEY = new Key('/local/pins') +// const WALK_DAG_CONCURRENCY_LIMIT = 300 +// const IS_PINNED_WITH_TYPE_CONCURRENCY_LIMIT = 300 +// const PIN_DS_KEY = new Key('/local/pins') function invalidPinTypeErr (type) { const errMsg = `Invalid type '${type}', must be one of {direct, indirect, recursive, all}` return errCode(new Error(errMsg), 'ERR_INVALID_PIN_TYPE') } +function toKey (cid) { + return '/' + cid.multihash.toString('base64') +} + const PinTypes = { direct: 'direct', recursive: 'recursive', @@ -38,245 +36,201 @@ class PinManager { this.repo = repo this.dag = dag this.log = debug('ipfs:pin') - this.pinset = createPinSet(dag) this.directPins = new Set() this.recursivePins = new Set() } - async _walkDag ({ cid, preload = false, onCid = () => {} }) { - if (!CID.isCID(cid)) { - cid = new CID(cid) + async * _walkDag (cid, { preload = false }) { + const { value: node } = await this.dag.get(cid, { preload }) + + if (cid.codec === 'dag-pb') { + for (const link of node.Links) { + yield link.Hash + yield * this._walkDag(link.Hash, { preload }) + } + } else if (cid.codec === 'dag-cbor') { + for (const [_, childCid] of dagCborLinks(node)) { // eslint-disable-line no-unused-vars + yield childCid + yield * this._walkDag(childCid, { preload }) + } } + } - const walk = (cid) => { - return async () => { - const { value: node } = await this.dag.get(cid, { preload }) + async pinDirectly (cid, options = {}) { + await this.dag.get(cid, options) - onCid(cid) + return this.repo.pins.put(toKey(cid), cbor.encode({ + cid: cid.buffer, + type: PinTypes.direct, + name: options.name + })) + } - if (cid.codec === 'dag-pb') { - queue.addAll( - node.Links.map(link => walk(link.Hash)) - ) - } else if (cid.codec === 'dag-cbor') { - for (const [_, childCid] of dagCborLinks(node)) { // eslint-disable-line no-unused-vars - queue.add(walk(childCid)) - } - } + async unpin (cid) { + if (typeof cid === 'string' || cid instanceof String) { + // find pin with passed name + const result = await first(this.repo.pins.query({ + filters: [entry => { + const pin = cbor.decode(entry.value) + + return pin.name === cid + }], + limit: 1 + })) + + if (!result) { + // no pin with this name + return } - } - const queue = new Queue({ - concurrency: WALK_DAG_CONCURRENCY_LIMIT - }) - queue.add(walk(cid)) + cid = new CID(cbor.decode(result.value).cid) + } - await queue.onIdle() + return this.repo.pins.delete(toKey(cid)) } - directKeys () { - return Array.from(this.directPins, key => new CID(key).buffer) - } + async pinRecursively (cid, options = {}) { + await this.fetchCompleteDag(cid, options) - recursiveKeys () { - return Array.from(this.recursivePins, key => new CID(key).buffer) + await this.repo.pins.put(toKey(cid), cbor.encode({ + cid: cid.buffer, + type: PinTypes.recursive, + name: options.name + })) } - async getIndirectKeys ({ preload }) { - const indirectKeys = new Set() + async * directKeys () { + for await (const entry of this.repo.pins.query({ + filters: [(entry) => { + const pin = cbor.decode(entry.value) - for (const multihash of this.recursiveKeys()) { - await this._walkDag({ - cid: new CID(multihash), - preload: preload || false, - onCid: (cid) => { - cid = cid.toString() + return pin.type === PinTypes.direct + }] + })) { + const pin = cbor.decode(entry.value) - // recursive pins pre-empt indirect pins - if (!this.recursivePins.has(cid)) { - indirectKeys.add(cid) - } - } - }) + yield { + cid: new CID(pin.cid), + name: pin.name + } } - - return Array.from(indirectKeys) } - // Encode and write pin key sets to the datastore: - // a DAGLink for each of the recursive and direct pinsets - // a DAGNode holding those as DAGLinks, a kind of root pin - async flushPins () { - const [ - dLink, - rLink - ] = await Promise.all([ - // create a DAGLink to the node with direct pins - this.pinset.storeSet(this.directKeys()) - .then((result) => { - return new DAGLink(PinTypes.direct, result.node.size, result.cid) - }), - // create a DAGLink to the node with recursive pins - this.pinset.storeSet(this.recursiveKeys()) - .then((result) => { - return new DAGLink(PinTypes.recursive, result.node.size, result.cid) - }), - // the pin-set nodes link to a special 'empty' node, so make sure it exists - this.dag.put(new DAGNode(Buffer.alloc(0)), { - version: 0, - format: multicodec.DAG_PB, - hashAlg: multicodec.SHA2_256, - preload: false - }) - ]) - - // create a root node with DAGLinks to the direct and recursive DAGs - const rootNode = new DAGNode(Buffer.alloc(0), [dLink, rLink]) - const rootCid = await this.dag.put(rootNode, { - version: 0, - format: multicodec.DAG_PB, - hashAlg: multicodec.SHA2_256, - preload: false - }) - - // save root to datastore under a consistent key - await this.repo.datastore.put(PIN_DS_KEY, rootCid.buffer) - - this.log(`Flushed pins with root: ${rootCid}`) - } + async * recursiveKeys () { + for await (const entry of this.repo.pins.query({ + filters: [(entry) => { + const pin = cbor.decode(entry.value) - async load () { - const has = await this.repo.datastore.has(PIN_DS_KEY) + return pin.type === PinTypes.recursive + }] + })) { + const pin = cbor.decode(entry.value) - if (!has) { - return + yield { + cid: new CID(pin.cid), + name: pin.name + } } + } - const mh = await this.repo.datastore.get(PIN_DS_KEY) - const pinRoot = await this.dag.get(new CID(mh), '', { preload: false }) - - const [ - rKeys, dKeys - ] = await Promise.all([ - this.pinset.loadSet(pinRoot.value, PinTypes.recursive), - this.pinset.loadSet(pinRoot.value, PinTypes.direct) - ]) - - this.directPins = new Set(dKeys.map(k => cidToString(k))) - this.recursivePins = new Set(rKeys.map(k => cidToString(k))) + async * indirectKeys ({ preload }) { + for await (const { cid } of this.recursiveKeys()) { + for await (const childCid of this._walkDag(cid, { preload })) { + // recursive pins override indirect pins + const types = [ + PinTypes.recursive + ] - this.log('Loaded pins from the datastore') - } + const result = await this.isPinnedWithType(childCid, types) - async isPinnedWithType (multihash, type) { - const key = cidToString(multihash) - const { recursive, direct, all } = PinTypes + if (result.pinned) { + continue + } - // recursive - if ((type === recursive || type === all) && this.recursivePins.has(key)) { - return { - key, - pinned: true, - reason: recursive + yield childCid } } + } - if (type === recursive) { - return { - key, - pinned: false - } + async isPinnedWithType (cid, types) { + if (!Array.isArray(types)) { + types = [types] } - // direct - if ((type === direct || type === all) && this.directPins.has(key)) { - return { - key, - pinned: true, - reason: direct - } - } + const all = types.includes(PinTypes.all) + const direct = types.includes(PinTypes.direct) + const recursive = types.includes(PinTypes.recursive) + const indirect = types.includes(PinTypes.indirect) - if (type === direct) { - return { - key, - pinned: false - } - } + if (recursive || direct || all) { + const result = await first(this.repo.pins.query({ + prefix: toKey(cid), + filters: [entry => { + const pin = cbor.decode(entry.value) - // indirect (default) - // check each recursive key to see if multihash is under it - // arbitrary limit, enables handling 1000s of pins. - const queue = new Queue({ - concurrency: IS_PINNED_WITH_TYPE_CONCURRENCY_LIMIT - }) - let cid - - queue.addAll( - this.recursiveKeys() - .map(childKey => { - childKey = new CID(childKey) - - return async () => { - const has = await this.pinset.hasDescendant(childKey, key) - - if (has) { - cid = childKey - queue.clear() - } + if (all) { + return true } - }) - ) - await queue.onIdle() + return types.includes(pin.type) + }], + limit: 1 + })) - return { - key, - pinned: Boolean(cid), - reason: cid - } - } + if (result) { + const pin = cbor.decode(result.value) - // Gets CIDs of blocks used internally by the pinner - async getInternalBlocks () { - let mh + return { + cid, + pinned: true, + reason: pin.type + } + } + } - try { - mh = await this.repo.datastore.get(PIN_DS_KEY) - } catch (err) { - if (err.code === ERR_NOT_FOUND) { - this.log('No pinned blocks') + const self = this - return [] + async function * findChild (key, source) { + for await (const { cid: parentCid } of source) { + for await (const childCid of self._walkDag(parentCid, { preload: false })) { + if (childCid.equals(key)) { + yield parentCid + return + } + } } - - throw new Error(`Could not get pin sets root from datastore: ${err.message}`) } - const cid = new CID(mh) - const obj = await this.dag.get(cid, '', { preload: false }) + if (all || indirect) { + // indirect (default) + // check each recursive key to see if multihash is under it + + const parentCid = await first(findChild(cid, this.recursiveKeys())) - // The pinner stores an object that has two links to pin sets: - // 1. The directly pinned CIDs - // 2. The recursively pinned CIDs - // If large enough, these pin sets may have links to buckets to hold - // the pins - const cids = await this.pinset.getInternalCids(obj.value) + if (parentCid) { + return { + cid, + pinned: true, + reason: PinTypes.indirect, + parent: parentCid + } + } + } - return cids.concat(cid) + return { + cid, + pinned: false + } } async fetchCompleteDag (cid, options) { - await this._walkDag({ - cid, - preload: options.preload - }) + await all(this._walkDag(cid, { preload: options.preload })) } - // Returns an error if the pin type is invalid + // Throws an error if the pin type is invalid static checkPinType (type) { if (typeof type !== 'string' || !Object.keys(PinTypes).includes(type)) { - return invalidPinTypeErr(type) + throw invalidPinTypeErr(type) } } } diff --git a/packages/ipfs/src/core/components/pin/pin-set.js b/packages/ipfs/src/core/components/pin/pin-set.js deleted file mode 100644 index 552fc23313..0000000000 --- a/packages/ipfs/src/core/components/pin/pin-set.js +++ /dev/null @@ -1,317 +0,0 @@ -'use strict' - -const multihashes = require('multihashes') -const CID = require('cids') -const protobuf = require('protons') -const fnv1a = require('fnv1a') -const varint = require('varint') -const { DAGNode, DAGLink } = require('ipld-dag-pb') -const multicodec = require('multicodec') -const { default: Queue } = require('p-queue') -const dagCborLinks = require('dag-cbor-links') -const log = require('debug')('ipfs:pin:pin-set') -const pbSchema = require('./pin.proto') - -const emptyKeyHash = 'QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n' -const emptyKey = multihashes.fromB58String(emptyKeyHash) -const defaultFanout = 256 -const maxItems = 8192 -const pb = protobuf(pbSchema) - -const HAS_DESCENDANT_CONCURRENCY = 100 - -function toB58String (hash) { - return new CID(hash).toBaseEncodedString() -} - -function readHeader (rootNode) { - // rootNode.data should be a buffer of the format: - // < varint(headerLength) | header | itemData... > - const rootData = rootNode.Data - const hdrLength = varint.decode(rootData) - const vBytes = varint.decode.bytes - - if (vBytes <= 0) { - throw new Error('Invalid Set header length') - } - - if (vBytes + hdrLength > rootData.length) { - throw new Error('Impossibly large set header length') - } - - const hdrSlice = rootData.slice(vBytes, hdrLength + vBytes) - const header = pb.Set.decode(hdrSlice) - - if (header.version !== 1) { - throw new Error(`Unsupported Set version: ${header.version}`) - } - - if (header.fanout > rootNode.Links.length) { - throw new Error('Impossibly large fanout') - } - - return { - header: header, - data: rootData.slice(hdrLength + vBytes) - } -} - -function hash (seed, key) { - const buf = Buffer.alloc(4) - buf.writeUInt32LE(seed, 0) - const data = Buffer.concat([ - buf, Buffer.from(toB58String(key)) - ]) - return fnv1a(data.toString('binary')) -} - -function * cborCids (node) { - for (const [_, cid] of dagCborLinks(node)) { // eslint-disable-line no-unused-vars - yield cid - } -} - -exports = module.exports = function (dag) { - const pinSet = { - // should this be part of `object` API? - hasDescendant: async (parentCid, childhash) => { - if (parentCid.codec !== 'dag-pb' && parentCid.codec !== 'dag-cbor') { - return false - } - - const { value: root } = await dag.get(parentCid, { preload: false }) - const queue = new Queue({ - concurrency: HAS_DESCENDANT_CONCURRENCY - }) - - if (CID.isCID(childhash) || Buffer.isBuffer(childhash)) { - childhash = toB58String(childhash) - } - - let found = false - const seen = {} - - function searchChild (linkCid) { - return async () => { - if (found) { - return - } - - try { - const { value: childNode } = await dag.get(linkCid, { preload: false }) - - searchChildren(linkCid, childNode) - } catch (err) { - log(err) - } - } - } - - function searchChildren (cid, node) { - let links = [] - - if (cid.codec === 'dag-pb') { - links = node.Links - } else if (cid.codec === 'dag-cbor') { - links = cborCids(node) - } - - for (const link of links) { - const linkCid = cid.codec === 'dag-pb' ? link.Hash : link[1] - const bs58Link = toB58String(linkCid) - - if (bs58Link === childhash) { - queue.clear() - found = true - - return - } - - if (seen[bs58Link]) { - continue - } - - seen[bs58Link] = true - - if (linkCid.codec !== 'dag-pb' && linkCid.codec !== 'dag-cbor') { - continue - } - - queue.add(searchChild(linkCid)) - } - } - - searchChildren(parentCid, root) - - await queue.onIdle() - - return found - }, - - storeSet: async (keys) => { - const pins = keys.map(key => { - if (typeof key === 'string' || Buffer.isBuffer(key)) { - key = new CID(key) - } - - return { - key: key, - data: null - } - }) - - const rootNode = await pinSet.storeItems(pins) - const cid = await dag.put(rootNode, { - version: 0, - format: multicodec.DAG_PB, - hashAlg: multicodec.SHA2_256, - preload: false - }) - - return { - node: rootNode, - cid - } - }, - - storeItems: async (items) => { // eslint-disable-line require-await - return storePins(items, 0) - - async function storePins (pins, depth) { - const pbHeader = pb.Set.encode({ - version: 1, - fanout: defaultFanout, - seed: depth - }) - const headerBuf = Buffer.concat([ - Buffer.from(varint.encode(pbHeader.length)), pbHeader - ]) - const fanoutLinks = [] - - for (let i = 0; i < defaultFanout; i++) { - fanoutLinks.push(new DAGLink('', 1, emptyKey)) - } - - if (pins.length <= maxItems) { - const nodes = pins - .map(item => { - return ({ - link: new DAGLink('', 1, item.key), - data: item.data || Buffer.alloc(0) - }) - }) - // sorting makes any ordering of `pins` produce the same DAGNode - .sort((a, b) => Buffer.compare(a.link.Hash.buffer, b.link.Hash.buffer)) - - const rootLinks = fanoutLinks.concat(nodes.map(item => item.link)) - const rootData = Buffer.concat( - [headerBuf].concat(nodes.map(item => item.data)) - ) - - return new DAGNode(rootData, rootLinks) - } else { - // If the array of pins is > maxItems, we: - // - distribute the pins among `defaultFanout` bins - // - create a DAGNode for each bin - // - add each pin as a DAGLink to that bin - // - create a root DAGNode - // - add each bin as a DAGLink - // - send that root DAGNode via callback - // (using go-ipfs' "wasteful but simple" approach for consistency) - // https://github.com/ipfs/go-ipfs/blob/master/pin/set.go#L57 - - const bins = pins.reduce((bins, pin) => { - const n = hash(depth, pin.key) % defaultFanout - bins[n] = n in bins ? bins[n].concat([pin]) : [pin] - return bins - }, []) - - let idx = 0 - for (const bin of bins) { - const child = await storePins(bin, depth + 1) - - await storeChild(child, idx) - - idx++ - } - - return new DAGNode(headerBuf, fanoutLinks) - } - - async function storeChild (child, binIdx) { - const opts = { - version: 0, - format: multicodec.DAG_PB, - hashAlg: multicodec.SHA2_256, - preload: false - } - - const cid = await dag.put(child, opts) - - fanoutLinks[binIdx] = new DAGLink('', child.size, cid) - } - } - }, - - loadSet: async (rootNode, name) => { - const link = rootNode.Links.find(l => l.Name === name) - - if (!link) { - throw new Error('No link found with name ' + name) - } - - const res = await dag.get(link.Hash, '', { preload: false }) - const keys = [] - const stepPin = link => keys.push(link.Hash) - - await pinSet.walkItems(res.value, { stepPin }) - - return keys - }, - - walkItems: async (node, { stepPin = () => {}, stepBin = () => {} }) => { - const pbh = readHeader(node) - let idx = 0 - - for (const link of node.Links) { - if (idx < pbh.header.fanout) { - // the first pbh.header.fanout links are fanout bins - // if a fanout bin is not 'empty', dig into and walk its DAGLinks - const linkHash = link.Hash.buffer - - if (!emptyKey.equals(linkHash)) { - stepBin(link, idx, pbh.data) - - // walk the links of this fanout bin - const res = await dag.get(linkHash, '', { preload: false }) - - await pinSet.walkItems(res.value, { stepPin, stepBin }) - } - } else { - // otherwise, the link is a pin - stepPin(link, idx, pbh.data) - } - - idx++ - } - }, - - getInternalCids: async (rootNode) => { - // "Empty block" used by the pinner - const cids = [new CID(emptyKey)] - const stepBin = link => cids.push(link.Hash) - - for (const topLevelLink of rootNode.Links) { - cids.push(topLevelLink.Hash) - - const res = await dag.get(topLevelLink.Hash, '', { preload: false }) - - await pinSet.walkItems(res.value, { stepBin }) - } - - return cids - } - } - - return pinSet -} diff --git a/packages/ipfs/src/core/components/pin/pin.proto.js b/packages/ipfs/src/core/components/pin/pin.proto.js deleted file mode 100644 index 8e94fd8f52..0000000000 --- a/packages/ipfs/src/core/components/pin/pin.proto.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict' - -/** - * Protobuf interface - * from go-ipfs/pin/internal/pb/header.proto - */ -module.exports = ` - syntax = "proto2"; - - package ipfs.pin; - - option go_package = "pb"; - - message Set { - optional uint32 version = 1; - optional uint32 fanout = 2; - optional fixed32 seed = 3; - } -` diff --git a/packages/ipfs/src/core/components/pin/rm.js b/packages/ipfs/src/core/components/pin/rm.js index 5082c7eca2..3a159ad667 100644 --- a/packages/ipfs/src/core/components/pin/rm.js +++ b/packages/ipfs/src/core/components/pin/rm.js @@ -1,62 +1,51 @@ 'use strict' -const errCode = require('err-code') -const multibase = require('multibase') -const { parallelMap, collect } = require('streaming-iterables') -const pipe = require('it-pipe') const { resolvePath } = require('../../utils') const { PinTypes } = require('./pin-manager') -const PIN_RM_CONCURRENCY = 8 - module.exports = ({ pinManager, gcLock, dag }) => { - return async function rm (paths, options) { + return async function * rm (paths, options) { options = options || {} - const recursive = options.recursive !== false - - if (options.cidBase && !multibase.names.includes(options.cidBase)) { - throw errCode(new Error('invalid multibase'), 'ERR_INVALID_MULTIBASE') - } + const recursive = options.recursive == null ? true : options.recursive + const cids = await resolvePath(dag, paths, { signal: options.signal }) - const cids = await resolvePath(dag, paths) const release = await gcLock.readLock() try { // verify that each hash can be unpinned - const results = await pipe( - cids, - parallelMap(PIN_RM_CONCURRENCY, async cid => { - const { pinned, reason } = await pinManager.isPinnedWithType(cid, PinTypes.all) + for (const cid of cids) { + const { pinned, reason } = await pinManager.isPinnedWithType(cid, PinTypes.all) - if (!pinned) { - throw new Error(`${cid} is not pinned`) - } - if (reason !== PinTypes.recursive && reason !== PinTypes.direct) { - throw new Error(`${cid} is pinned indirectly under ${reason}`) - } - if (reason === PinTypes.recursive && !recursive) { - throw new Error(`${cid} is pinned recursively`) - } + if (!pinned) { + throw new Error(`${cid} is not pinned`) + } - return cid - }), - collect - ) + switch (reason) { + case (PinTypes.recursive): + if (!recursive) { + throw new Error(`${cid} is pinned recursively`) + } - // update the pin sets in memory - results.forEach(cid => { - if (recursive && pinManager.recursivePins.has(cid.toString())) { - pinManager.recursivePins.delete(cid.toString()) - } else { - pinManager.directPins.delete(cid.toString()) - } - }) + await pinManager.unpin(cid) + + yield { + cid + } + + break + case (PinTypes.direct): + await pinManager.unpin(cid) - // persist updated pin sets to datastore - await pinManager.flushPins() + yield { + cid + } - return results.map(cid => ({ cid })) + break + default: + throw new Error(`${cid} is pinned indirectly under ${reason}`) + } + } } finally { release() } diff --git a/packages/ipfs/src/core/components/repo/gc.js b/packages/ipfs/src/core/components/repo/gc.js index 3f19789f37..3e6dd1b23c 100644 --- a/packages/ipfs/src/core/components/repo/gc.js +++ b/packages/ipfs/src/core/components/repo/gc.js @@ -13,7 +13,7 @@ const { parallelMerge, transform, map } = require('streaming-iterables') const BLOCK_RM_CONCURRENCY = 256 // Perform mark and sweep garbage collection -module.exports = ({ gcLock, pin, pinManager, refs, repo }) => { +module.exports = ({ gcLock, pin, refs, repo }) => { return async function * gc () { const start = Date.now() log('Creating set of marked blocks') @@ -22,7 +22,7 @@ module.exports = ({ gcLock, pin, pinManager, refs, repo }) => { try { // Mark all blocks that are being used - const markedSet = await createMarkedSet({ pin, pinManager, refs, repo }) + const markedSet = await createMarkedSet({ pin, refs, repo }) // Get all blocks keys from the blockstore const blockKeys = repo.blocks.query({ keysOnly: true }) @@ -37,14 +37,9 @@ module.exports = ({ gcLock, pin, pinManager, refs, repo }) => { } // Get Set of CIDs of blocks to keep -async function createMarkedSet ({ pin, pinManager, refs, repo }) { +async function createMarkedSet ({ pin, refs, repo }) { const pinsSource = map(({ cid }) => cid, pin.ls()) - const pinInternalsSource = (async function * () { - const cids = await pinManager.getInternalBlocks() - yield * cids - })() - const mfsSource = (async function * () { let mh try { @@ -66,7 +61,7 @@ async function createMarkedSet ({ pin, pinManager, refs, repo }) { })() const output = new Set() - for await (const cid of parallelMerge(pinsSource, pinInternalsSource, mfsSource)) { + for await (const cid of parallelMerge(pinsSource, mfsSource)) { output.add(cidToString(cid, { base: 'base32' })) } return output diff --git a/packages/ipfs/src/core/components/start.js b/packages/ipfs/src/core/components/start.js index ec771cb09b..891cdd9ec9 100644 --- a/packages/ipfs/src/core/components/start.js +++ b/packages/ipfs/src/core/components/start.js @@ -229,7 +229,7 @@ function createApi ({ pubsub, refs, repo: { - gc: Components.repo.gc({ gcLock, pin, pinManager, refs, repo }), + gc: Components.repo.gc({ gcLock, pin, refs, repo }), stat: Components.repo.stat({ repo }), version: Components.repo.version({ repo }) }, diff --git a/packages/ipfs/src/http/api/resources/dag.js b/packages/ipfs/src/http/api/resources/dag.js index 8b6ff198ce..505c0ec8c1 100644 --- a/packages/ipfs/src/http/api/resources/dag.js +++ b/packages/ipfs/src/http/api/resources/dag.js @@ -14,6 +14,7 @@ const { const all = require('it-all') const log = debug('ipfs:http-api:dag') log.error = debug('ipfs:http-api:dag:error') +const drain = require('it-drain') const IpldFormats = { get [multicodec.RAW] () { @@ -252,7 +253,7 @@ exports.put = { } if (request.query.pin) { - await ipfs.pin.add(cid) + await drain(ipfs.pin.add(cid)) } return h.response({ diff --git a/packages/ipfs/src/http/api/resources/pin.js b/packages/ipfs/src/http/api/resources/pin.js index c853f9bdda..4d2e860307 100644 --- a/packages/ipfs/src/http/api/resources/pin.js +++ b/packages/ipfs/src/http/api/resources/pin.js @@ -9,6 +9,7 @@ const pipe = require('it-pipe') const ndjson = require('iterable-ndjson') const { cidToString } = require('../../../utils/cid') const streamResponse = require('../../utils/stream-response') +const all = require('it-all') function parseArgs (request, h) { let { arg } = request.query @@ -93,7 +94,7 @@ exports.add = { let result try { - result = await ipfs.pin.add(path, { recursive }) + result = await all(ipfs.pin.add(path, { recursive })) } catch (err) { if (err.message.includes('already pinned recursively')) { throw Boom.boomify(err, { statusCode: 400 }) @@ -122,7 +123,7 @@ exports.rm = { let result try { - result = await ipfs.pin.rm(path, { recursive }) + result = await all(ipfs.pin.rm(path, { recursive })) } catch (err) { throw Boom.boomify(err, { message: 'Failed to remove pin' }) } diff --git a/packages/ipfs/test/cli/pin.js b/packages/ipfs/test/cli/pin.js index b65e75a8bf..b667221799 100644 --- a/packages/ipfs/test/cli/pin.js +++ b/packages/ipfs/test/cli/pin.js @@ -33,7 +33,7 @@ describe('pin', () => { describe('rm', function () { it('recursively (default)', async () => { - ipfs.pin.rm.withArgs([pins.root], { recursive: true }).resolves([{ + ipfs.pin.rm.withArgs([pins.root], { recursive: true }).returns([{ cid: new CID(pins.root) }]) @@ -42,7 +42,7 @@ describe('pin', () => { }) it('non recursively', async () => { - ipfs.pin.rm.withArgs([pins.root], { recursive: false }).resolves([{ + ipfs.pin.rm.withArgs([pins.root], { recursive: false }).returns([{ cid: new CID(pins.root) }]) @@ -51,7 +51,7 @@ describe('pin', () => { }) it('non recursively (short option)', async () => { - ipfs.pin.rm.withArgs([pins.root], { recursive: false }).resolves([{ + ipfs.pin.rm.withArgs([pins.root], { recursive: false }).returns([{ cid: new CID(pins.root) }]) @@ -60,7 +60,7 @@ describe('pin', () => { }) it('should rm and print CIDs encoded in specified base', async () => { - ipfs.pin.rm.withArgs([pins.root], { recursive: true }).resolves([{ + ipfs.pin.rm.withArgs([pins.root], { recursive: true }).returns([{ cid: new CID(pins.root) }]) @@ -72,7 +72,7 @@ describe('pin', () => { describe('add', function () { it('recursively (default)', async () => { - ipfs.pin.add.withArgs([pins.root], { recursive: true }).resolves([{ + ipfs.pin.add.withArgs([pins.root], { recursive: true }).returns([{ cid: new CID(pins.root) }]) @@ -81,7 +81,7 @@ describe('pin', () => { }) it('non recursively', async () => { - ipfs.pin.add.withArgs([pins.root], { recursive: false }).resolves([{ + ipfs.pin.add.withArgs([pins.root], { recursive: false }).returns([{ cid: new CID(pins.root) }]) @@ -90,7 +90,7 @@ describe('pin', () => { }) it('non recursively (short option)', async () => { - ipfs.pin.add.withArgs([pins.root], { recursive: false }).resolves([{ + ipfs.pin.add.withArgs([pins.root], { recursive: false }).returns([{ cid: new CID(pins.root) }]) @@ -99,7 +99,7 @@ describe('pin', () => { }) it('should rm and print CIDs encoded in specified base', async () => { - ipfs.pin.add.withArgs([pins.root], { recursive: true }).resolves([{ + ipfs.pin.add.withArgs([pins.root], { recursive: true }).returns([{ cid: new CID(pins.root) }]) diff --git a/packages/ipfs/test/core/gc.spec.js b/packages/ipfs/test/core/gc.spec.js index 90b0ebf982..11b7266c38 100644 --- a/packages/ipfs/test/core/gc.spec.js +++ b/packages/ipfs/test/core/gc.spec.js @@ -4,6 +4,7 @@ const { expect } = require('interface-ipfs-core/src/utils/mocha') const last = require('it-last') +const drain = require('it-drain') const factory = require('../utils/factory') const pEvent = require('p-event') @@ -199,7 +200,7 @@ describe.skip('gc', function () { // Pin first block // Note: pin add will take a read lock const pinLockRequested = pEvent(lockEmitter, 'readLock request') - const pin1 = ipfs.pin.add(cid1) + const pin1 = last(ipfs.pin.add(cid1)) // Once pin lock has been requested, start GC await pinLockRequested @@ -209,7 +210,7 @@ describe.skip('gc', function () { // TODO: Adding pin for removed block never returns, which means the lock // never gets released - // const pin2 = ipfs.pin.add(cid2) + // const pin2 = last(ipfs.pin.add(cid2)) // Confirm second second block has been removed const localRefs = (await ipfs.refs.local()).map(r => r.ref) @@ -230,13 +231,13 @@ describe.skip('gc', function () { const cid2 = (await ipfs.block.put(Buffer.from('block to pin rm 2'), null)).cid // Pin blocks - await ipfs.pin.add(cid1) - await ipfs.pin.add(cid2) + await drain(ipfs.pin.add(cid1)) + await drain(ipfs.pin.add(cid2)) // Unpin first block // Note: pin rm will take a read lock const pinLockRequested = pEvent(lockEmitter, 'readLock request') - const pinRm1 = ipfs.pin.rm(cid1) + const pinRm1 = last(ipfs.pin.rm(cid1)) // Once pin lock has been requested, start GC await pinLockRequested @@ -246,7 +247,7 @@ describe.skip('gc', function () { // Once GC has started, start second pin rm await gcStarted - const pinRm2 = ipfs.pin.rm(cid2) + const pinRm2 = last(ipfs.pin.rm(cid2)) const deleted = (await gc).map(i => i.cid.toString()) await pinRm1 diff --git a/packages/ipfs/test/core/node.js b/packages/ipfs/test/core/node.js index f7aa9bde93..bb95a6d65e 100644 --- a/packages/ipfs/test/core/node.js +++ b/packages/ipfs/test/core/node.js @@ -1,6 +1,4 @@ 'use strict' require('./name-pubsub') -require('./pin') -require('./pin-set') require('./utils') diff --git a/packages/ipfs/test/core/pin-set.js b/packages/ipfs/test/core/pin-set.js deleted file mode 100644 index bec62bb962..0000000000 --- a/packages/ipfs/test/core/pin-set.js +++ /dev/null @@ -1,188 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const { util, DAGNode } = require('ipld-dag-pb') -const CID = require('cids') -const map = require('p-map') -const IPFS = require('../../src/core') -const createPinSet = require('../../src/core/components/pin/pin-set') -const createTempRepo = require('../utils/create-repo-nodejs') - -const emptyKeyHash = 'QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n' -const defaultFanout = 256 -const maxItems = 8192 - -/** - * Creates @param num DAGNodes, limited to 500 at a time to save memory - * @param {[type]} num the number of nodes to create - * @return {Promise>} - */ -function createNodes (num) { - return map(Array.from(Array(num)), (_, i) => createNode(String(i)), { concurrency: 500 }) -} - -async function createNode (data, links = []) { - const node = new DAGNode(data, links) - const cid = await util.cid(util.serialize(node), { cidVersion: 0 }) - return { node, cid } -} - -describe('pinSet', function () { - let ipfs - let pinSet - let repo - - before(async function () { - this.timeout(80 * 1000) - repo = createTempRepo() - ipfs = await IPFS.create({ - silent: true, - repo, - config: { - Bootstrap: [], - Discovery: { - MDNS: { - Enabled: false - } - } - }, - preload: { enabled: false } - }) - pinSet = createPinSet(ipfs.dag) - }) - - after(function () { - this.timeout(80 * 1000) - return ipfs.stop() - }) - - after(() => repo.teardown()) - - describe('storeItems', function () { - it('generates a root node with links and hash', async function () { - const expectedRootHash = 'QmcLiSTjcjoVC2iuGbk6A2PVcWV3WvjZT4jxfNis1vjyrR' - - const result = await createNode('data') - const nodeHash = result.cid.toBaseEncodedString() - const rootNode = await pinSet.storeSet([nodeHash]) - - expect(rootNode.cid.toBaseEncodedString()).to.eql(expectedRootHash) - expect(rootNode.node.Links).to.have.length(defaultFanout + 1) - - const lastLink = rootNode.node.Links[rootNode.node.Links.length - 1] - const mhash = lastLink.Hash.toBaseEncodedString() - expect(mhash).to.eql(nodeHash) - }) - }) - - describe('handles large sets', function () { - it('handles storing items > maxItems', async function () { - this.timeout(90 * 1000) - const expectedHash = 'QmbvhSy83QWfgLXDpYjDmLWBFfGc8utoqjcXHyj3gYuasT' - const count = maxItems + 1 - const nodes = await createNodes(count) - const result = await pinSet.storeSet(nodes.map(n => n.cid)) - - expect(result.node.size).to.eql(3184696) - expect(result.node.Links).to.have.length(defaultFanout) - expect(result.cid.toBaseEncodedString()).to.eql(expectedHash) - - const loaded = await pinSet.loadSet(result.node, '') - expect(loaded).to.have.length(30) - - const hashes = loaded.map(l => new CID(l).toBaseEncodedString()) - - // just check the first node, assume all are children if successful - const has = await pinSet.hasDescendant(result.cid, hashes[0]) - expect(has).to.eql(true) - }) - - // This test is largely taken from go-ipfs/pin/set_test.go - // It fails after reaching maximum call stack depth but I don't believe it's - // infinite. We need to reference go's pinSet impl to make sure - // our sharding behaves correctly, or perhaps this test is misguided - // - // FIXME: Update: AS 2020-01-14 this test currently is failing with: - // - // TypeError: Cannot read property 'length' of undefined - // at storePins (src/core/components/pin/pin-set.js:195:18) - // at storePins (src/core/components/pin/pin-set.js:231:33) - // at storePins (src/core/components/pin/pin-set.js:231:33) - // at Object.storeItems (src/core/components/pin/pin-set.js:178:14) - // at Object.storeSet (src/core/components/pin/pin-set.js:163:37) - // at Context. (test/core/pin-set.js:116:39) - // at processTicksAndRejections (internal/process/task_queues.js:94:5) - it.skip('stress test: stores items > (maxItems * defaultFanout) + 1', async function () { - this.timeout(180 * 1000) - - // this value triggers the creation of a recursive shard. - // If the recursive sharding is done improperly, this will result in - // an infinite recursion and crash (OOM) - const limit = (defaultFanout * maxItems) + 1 - - const nodes = await createNodes(limit) - const rootNodes0 = await pinSet.storeSet(nodes.slice(0, -1).map(n => n.cid)) - const rootNodes1 = await pinSet.storeSet(nodes.map(n => n.cid)) - - expect(rootNodes0.length - rootNodes1.length).to.eql(2) - }) - }) - - describe('walkItems', function () { - it('fails if node doesn\'t have a pin-set protobuf header', async function () { - const { node } = await createNode('datum') - await expect(pinSet.walkItems(node, {})) - .to.eventually.be.rejected() - }) - - it('visits all links of a root node', async function () { - this.timeout(90 * 1000) - - const seenPins = [] - const stepPin = (link, idx, data) => seenPins.push({ link, idx, data }) - const seenBins = [] - const stepBin = (link, idx, data) => seenBins.push({ link, idx, data }) - - const nodes = await createNodes(maxItems + 1) - const result = await pinSet.storeSet(nodes.map(n => n.cid)) - - await pinSet.walkItems(result.node, { stepPin, stepBin }) - expect(seenPins).to.have.length(maxItems + 1) - expect(seenBins).to.have.length(defaultFanout) - }) - - it('visits all non-fanout links of a root node', async () => { - const seen = [] - const stepPin = (link, idx, data) => seen.push({ link, idx, data }) - - const nodes = await createNodes(defaultFanout) - const result = await pinSet.storeSet(nodes.map(n => n.cid)) - - await pinSet.walkItems(result.node, { stepPin }) - - expect(seen).to.have.length(defaultFanout) - expect(seen[0].idx).to.eql(defaultFanout) - - seen.forEach(item => { - expect(item.data).to.eql(Buffer.alloc(0)) - expect(item.link).to.exist() - }) - }) - }) - - describe('getInternalCids', function () { - it('gets all links and empty key CID', async () => { - const nodes = await createNodes(defaultFanout) - const result = await pinSet.storeSet(nodes.map(n => n.cid)) - - const rootNode = new DAGNode('pins', [{ Hash: result.cid }]) - const cids = await pinSet.getInternalCids(rootNode) - - expect(cids.length).to.eql(2) - const cidStrs = cids.map(c => c.toString()) - expect(cidStrs).includes(emptyKeyHash) - expect(cidStrs).includes(result.cid.toString()) - }) - }) -}) diff --git a/packages/ipfs/test/core/pin.js b/packages/ipfs/test/core/pin.js deleted file mode 100644 index 98da97b55c..0000000000 --- a/packages/ipfs/test/core/pin.js +++ /dev/null @@ -1,437 +0,0 @@ -/* eslint max-nested-callbacks: ["error", 8] */ -/* eslint-env mocha */ -'use strict' - -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const fs = require('fs') -const { - DAGNode -} = require('ipld-dag-pb') -const all = require('it-all') -const CID = require('cids') -const IPFS = require('../../src/core') -const createTempRepo = require('../utils/create-repo-nodejs') - -// fixture structure: -// planets/ -// solar-system.md -// mercury/ -// wiki.md -const pins = { - root: 'QmTAMavb995EHErSrKo7mB8dYkpaSJxu6ys1a6XJyB2sys', - solarWiki: 'QmTMbkDfvHwq3Aup6Nxqn3KKw9YnoKzcZvuArAfQ9GF3QG', - mercuryDir: 'QmbJCNKXJqVK8CzbjpNFz2YekHwh3CSHpBA86uqYg3sJ8q', - mercuryWiki: 'QmVgSHAdMxFAuMP2JiMAYkB8pCWP1tcB9djqvq8GKAFiHi' -} -const pinTypes = { - direct: 'direct', - recursive: 'recursive', - indirect: 'indirect', - all: 'all' -} - -describe('pin', function () { - const fixtures = [ - 'test/fixtures/planets/mercury/wiki.md', - 'test/fixtures/planets/solar-system.md' - ].map(path => ({ - path, - content: fs.readFileSync(path) - })) - - let ipfs - let pin - let repo - - async function isPinnedWithType (path, type) { - try { - for await (const _ of pin.ls(path, { type })) { // eslint-disable-line no-unused-vars - return true - } - return false - } catch (err) { - return false - } - } - - async function expectPinned (cid, type = pinTypes.all, pinned = true) { - if (typeof type === 'boolean') { - pinned = type - type = pinTypes.all - } - - const result = await isPinnedWithType(cid, type) - expect(result).to.eql(pinned) - } - - async function clearPins () { - for await (const { cid } of pin.ls({ type: pinTypes.recursive })) { - await pin.rm(cid) - } - - for await (const { cid } of pin.ls({ type: pinTypes.direct })) { - await pin.rm(cid) - } - } - - before(async function () { - this.timeout(20 * 1000) - repo = createTempRepo() - ipfs = await IPFS.create({ - silent: true, - repo, - config: { Bootstrap: [] }, - preload: { enabled: false } - }) - - pin = ipfs.pin - await all(ipfs.add(fixtures)) - }) - - after(function () { - this.timeout(60 * 1000) - return ipfs.stop() - }) - - after(() => repo.teardown()) - - describe('pinned status', function () { - beforeEach(async () => { - await clearPins() - await pin.add(pins.root) - }) - - it('should be pinned when added', async () => { - await pin.add(pins.solarWiki) - return expectPinned(pins.solarWiki) - }) - - it('should not be pinned when not in datastore', () => { - const falseHash = `${pins.root.slice(0, -2)}ss` - return expectPinned(falseHash, false) - }) - - it('should not be pinned when in datastore but not added', async () => { - await pin.rm(pins.root) - return expectPinned(pins.root, false) - }) - - it('should be pinned recursively when added', () => { - return expectPinned(pins.root, pinTypes.recursive) - }) - - it('should be pinned indirectly', () => { - return expectPinned(pins.mercuryWiki, pinTypes.indirect) - }) - - it('should be pinned directly', async () => { - await pin.add(pins.mercuryDir, { recursive: false }) - return expectPinned(pins.mercuryDir, pinTypes.direct) - }) - - it('should not be pinned when not in datastore or added', async () => { - await clearPins() - return expectPinned(pins.mercuryDir, pinTypes.direct, false) - }) - }) - - describe('add', function () { - beforeEach(function () { - return clearPins() - }) - - it('should add recursively', async () => { - await pin.add(pins.root) - await expectPinned(pins.root, pinTypes.recursive) - - const pinChecks = Object.values(pins).map(hash => expectPinned(hash)) - return Promise.all(pinChecks) - }) - - it('should add directly', async () => { - await pin.add(pins.root, { recursive: false }) - await Promise.all([ - expectPinned(pins.root, pinTypes.direct), - expectPinned(pins.solarWiki, false) - ]) - }) - - it('should recursively pin parent of direct pin', async () => { - await pin.add(pins.solarWiki, { recursive: false }) - await pin.add(pins.root) - await Promise.all([ - // solarWiki is pinned both directly and indirectly o.O - expectPinned(pins.solarWiki, pinTypes.direct), - expectPinned(pins.solarWiki, pinTypes.indirect) - ]) - }) - - it('should fail to directly pin a recursive pin', async () => { - await pin.add(pins.root) - return expect(pin.add(pins.root, { recursive: false })) - .to.eventually.be.rejected() - .with(/already pinned recursively/) - }) - - it('should fail to pin a hash not in datastore', function () { - this.timeout(5 * 1000) - const falseHash = `${pins.root.slice(0, -2)}ss` - return expect(pin.add(falseHash, { timeout: '2s' })) - .to.eventually.be.rejected() - .with.a.property('code').that.equals('ERR_TIMEOUT') - }) - - // TODO block rm breaks subsequent tests - // it.skip('needs all children in datastore to pin recursively', () => { - // return ipfs.block.rm(pins.mercuryWiki) - // .then(() => expectTimeout(pin.add(pins.root), 4000)) - // }) - }) - - describe('ls', function () { - before(async () => { - await clearPins() - await Promise.all([ - pin.add(pins.root), - pin.add(pins.mercuryDir, { recursive: false }) - ]) - }) - - it('should list pins of a particular CID', async () => { - const out = await all(pin.ls(pins.mercuryDir)) - expect(out[0].cid.toString()).to.eql(pins.mercuryDir) - }) - - it('should list indirect pins that supersede direct pins', async () => { - const ls = await all(pin.ls()) - const pinType = ls.find(out => out.cid.toString() === pins.mercuryDir).type - expect(pinType).to.eql(pinTypes.indirect) - }) - - it('should list all pins', async () => { - const out = await all(pin.ls()) - - expect(out).to.deep.include.members([ - { - type: 'recursive', - cid: new CID('QmTAMavb995EHErSrKo7mB8dYkpaSJxu6ys1a6XJyB2sys') - }, - { - type: 'indirect', - cid: new CID('QmTMbkDfvHwq3Aup6Nxqn3KKw9YnoKzcZvuArAfQ9GF3QG') - }, - { - type: 'indirect', - cid: new CID('QmbJCNKXJqVK8CzbjpNFz2YekHwh3CSHpBA86uqYg3sJ8q') - }, - { - type: 'indirect', - cid: new CID('QmVgSHAdMxFAuMP2JiMAYkB8pCWP1tcB9djqvq8GKAFiHi') - } - ]) - }) - - it('should list all direct pins', async () => { - const out = await all(pin.ls({ type: 'direct' })) - - expect(out).to.deep.include.members([ - { - type: 'direct', - cid: new CID('QmbJCNKXJqVK8CzbjpNFz2YekHwh3CSHpBA86uqYg3sJ8q') - } - ]) - }) - - it('should list all recursive pins', async () => { - const out = await all(pin.ls({ type: 'recursive' })) - - expect(out).to.deep.include.members([ - { - type: 'recursive', - cid: new CID('QmTAMavb995EHErSrKo7mB8dYkpaSJxu6ys1a6XJyB2sys') - } - ]) - }) - - it('should list all indirect pins', async () => { - const out = await all(pin.ls({ type: 'indirect' })) - - expect(out).to.deep.include.members([ - { - type: 'indirect', - cid: new CID('QmTMbkDfvHwq3Aup6Nxqn3KKw9YnoKzcZvuArAfQ9GF3QG') - }, - { - type: 'indirect', - cid: new CID('QmbJCNKXJqVK8CzbjpNFz2YekHwh3CSHpBA86uqYg3sJ8q') - }, - { - type: 'indirect', - cid: new CID('QmVgSHAdMxFAuMP2JiMAYkB8pCWP1tcB9djqvq8GKAFiHi') - } - ]) - }) - - it('should list direct pins for CID', async () => { - const out = await all(pin.ls(pins.mercuryDir, { type: 'direct' })) - - expect(out).to.have.deep.members([ - { - type: 'direct', - cid: new CID(pins.mercuryDir) - } - ]) - }) - - it('should list direct pins for path', async () => { - const out = await all(pin.ls(`/ipfs/${pins.root}/mercury/`, { type: 'direct' })) - - expect(out).to.have.deep.members([ - { - type: 'direct', - cid: new CID(pins.mercuryDir) - } - ]) - }) - - it('should list direct pins for path (no match)', () => { - return expect(all(pin.ls(`/ipfs/${pins.root}/mercury/wiki.md`, { type: 'direct' }))) - .to.eventually.be.rejected() - }) - - it('should list direct pins for CID (no match)', () => { - return expect(all(pin.ls(pins.root, { type: 'direct' }))) - .to.eventually.be.rejected() - }) - - it('should list recursive pins for CID', async () => { - const out = await all(pin.ls(pins.root, { type: 'recursive' })) - - expect(out).to.have.deep.members([ - { - type: 'recursive', - cid: new CID(pins.root) - } - ]) - }) - - it('should list recursive pins for CID (no match)', () => { - return expect(all(pin.ls(pins.mercuryDir, { type: 'recursive' }))) - .to.eventually.be.rejected() - }) - - it('should list indirect pins for CID', async () => { - const out = await all(pin.ls(pins.solarWiki, { type: 'indirect' })) - - expect(out).to.have.deep.members([ - { - type: `indirect through ${pins.root}`, - cid: new CID(pins.solarWiki) - } - ]) - }) - - it('should list indirect pins for CID (no match)', () => { - return expect(all(pin.ls(pins.root, { type: 'indirect' }))) - .to.eventually.be.rejected() - }) - }) - - describe('rm', function () { - beforeEach(async () => { - await clearPins() - await pin.add(pins.root) - }) - - it('should remove a recursive pin', async () => { - await pin.rm(pins.root) - await Promise.all([ - expectPinned(pins.root, false), - expectPinned(pins.mercuryWiki, false) - ]) - }) - - it('should remove a direct pin', async () => { - await clearPins() - await pin.add(pins.mercuryDir, { recursive: false }) - await pin.rm(pins.mercuryDir) - await expectPinned(pins.mercuryDir, false) - }) - - it('should fail to remove an indirect pin', async () => { - await expect(pin.rm(pins.solarWiki)) - .to.eventually.be.rejected() - .with(/is pinned indirectly under/) - await expectPinned(pins.solarWiki) - }) - - it('should fail when an item is not pinned', async () => { - await pin.rm(pins.root) - await expect(pin.rm(pins.root)) - .to.eventually.be.rejected() - .with(/is not pinned/) - }) - }) - - describe('non-dag-pb nodes', function () { - it('should pin dag-cbor', async () => { - const cid = await ipfs.dag.put({}, { - format: 'dag-cbor', - hashAlg: 'sha2-256' - }) - - await pin.add(cid) - - const pins = await all(pin.ls()) - - expect(pins).to.deep.include({ - type: 'recursive', - cid - }) - }) - - it('should pin raw', async () => { - const cid = await ipfs.dag.put(Buffer.alloc(0), { - format: 'raw', - hashAlg: 'sha2-256' - }) - - await pin.add(cid) - - const pins = await all(pin.ls()) - - expect(pins).to.deep.include({ - type: 'recursive', - cid - }) - }) - - it('should pin dag-cbor with dag-pb child', async () => { - const child = await ipfs.dag.put(new DAGNode(Buffer.alloc(0)), { - format: 'dag-pb', - hashAlg: 'sha2-256' - }) - const parent = await ipfs.dag.put({ - child - }, { - format: 'dag-cbor', - hashAlg: 'sha2-256' - }) - - await pin.add(parent, { - recursive: true - }) - - const pins = await all(pin.ls()) - - expect(pins).to.deep.include({ - cid: parent, - type: 'recursive' - }) - expect(pins).to.deep.include({ - cid: child, - type: 'indirect' - }) - }) - }) -}) diff --git a/packages/ipfs/test/core/pin.spec.js b/packages/ipfs/test/core/pin.spec.js deleted file mode 100644 index f1e2ba1ab8..0000000000 --- a/packages/ipfs/test/core/pin.spec.js +++ /dev/null @@ -1,34 +0,0 @@ -/* eslint max-nested-callbacks: ["error", 8] */ -/* eslint-env mocha */ -'use strict' - -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const all = require('it-all') -const factory = require('../utils/factory') - -describe('pin', function () { - this.timeout(10 * 1000) - const df = factory() - let ipfsd, ipfs - - before(async () => { - ipfsd = await df.spawn() - ipfs = ipfsd.api - }) - - after(() => df.clean()) - - describe('ls', () => { - it('should throw error for invalid non-string pin type option', () => { - return expect(all(ipfs.pin.ls({ type: 6 }))) - .to.eventually.be.rejected() - .with.property('code').that.equals('ERR_INVALID_PIN_TYPE') - }) - - it('should throw error for invalid string pin type option', () => { - return expect(all(ipfs.pin.ls({ type: '__proto__' }))) - .to.eventually.be.rejected() - .with.property('code').that.equals('ERR_INVALID_PIN_TYPE') - }) - }) -}) diff --git a/packages/ipfs/test/http-api/inject/pin.js b/packages/ipfs/test/http-api/inject/pin.js index f4a77ab1d6..bb1967e023 100644 --- a/packages/ipfs/test/http-api/inject/pin.js +++ b/packages/ipfs/test/http-api/inject/pin.js @@ -52,7 +52,14 @@ module.exports = (http) => { }) it('unpins recursive pins', async () => { - const res = await api.inject({ + let res = await api.inject({ + method: 'POST', + url: `/api/v0/pin/add?arg=${pins.root1}` + }) + + expect(res.statusCode).to.equal(200) + + res = await api.inject({ method: 'POST', url: `/api/v0/pin/rm?arg=${pins.root1}` }) From b65c94be8a823f37a791713c17ee2ccfe5883a85 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 4 Mar 2020 10:28:25 +0000 Subject: [PATCH 02/40] chore: revert update of ipfs-repo in mfs dev deps --- packages/ipfs-mfs/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs-mfs/package.json b/packages/ipfs-mfs/package.json index 5d96d455a0..62ca4f305d 100644 --- a/packages/ipfs-mfs/package.json +++ b/packages/ipfs-mfs/package.json @@ -48,7 +48,7 @@ "form-data": "^3.0.0", "ipfs-block": "^0.8.1", "ipfs-block-service": "^0.16.0", - "ipfs-repo": "github:ipfs/js-ipfs-repo#store-pins-in-datastore", + "ipfs-repo": "^0.30.1", "ipld": "^0.25.0", "memdown": "^5.1.0", "nyc": "^15.0.0", From a8e1a061a3bcaae4ed3240d84a5c2206dd8d00df Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 5 Mar 2020 17:21:37 +0000 Subject: [PATCH 03/40] chore: fix interop tests --- packages/interface-ipfs-core/SPEC/PIN.md | 10 +++---- packages/ipfs-http-client/src/pin/ls.js | 17 +++++++++-- packages/ipfs/package.json | 2 +- packages/ipfs/src/core/components/init.js | 10 +++++-- packages/ipfs/src/core/components/pin/ls.js | 14 ++++----- .../src/core/components/pin/pin-manager.js | 29 ++++--------------- packages/ipfs/src/http/api/resources/pin.js | 6 ++-- 7 files changed, 43 insertions(+), 45 deletions(-) diff --git a/packages/interface-ipfs-core/SPEC/PIN.md b/packages/interface-ipfs-core/SPEC/PIN.md index 1de696b088..20c2d4e696 100644 --- a/packages/interface-ipfs-core/SPEC/PIN.md +++ b/packages/interface-ipfs-core/SPEC/PIN.md @@ -21,7 +21,7 @@ Where: | Type | Description | | -------- | -------- | -| `AsyncIterable` | An async iterable that yields objects containing the CIDs that were pinned | +| `AsyncIterable<{ cid: CID }>` | An async iterable that yields objects containing the CIDs that were pinned | Each yielded object has the form: @@ -34,10 +34,10 @@ Each yielded object has the form: **Example:** ```JavaScript -const pinset = await ipfs.pin.add('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') -console.log(pinset) -// Logs: -// [ { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') } ] +for await (const pin of ipfs.pin.add('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')) { + console.log(pinset) +} +// { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') } ``` A great source of [examples][] can be found in the tests for this API. diff --git a/packages/ipfs-http-client/src/pin/ls.js b/packages/ipfs-http-client/src/pin/ls.js index f9e0968ace..d60ae441e3 100644 --- a/packages/ipfs-http-client/src/pin/ls.js +++ b/packages/ipfs-http-client/src/pin/ls.js @@ -5,6 +5,19 @@ const CID = require('cids') const configure = require('../lib/configure') const toIterable = require('stream-to-it/source') +function toPin (type, cid, comments) { + const pin = { + type, + cid: new CID(cid) + } + + if (comments) { + pin.comments = comments + } + + return pin +} + module.exports = configure(({ ky }) => { return async function * ls (path, options) { if (path && path.type) { @@ -31,11 +44,11 @@ module.exports = configure(({ ky }) => { for await (const pin of ndjson(toIterable(res.body))) { if (pin.Keys) { // non-streaming response for (const cid of Object.keys(pin.Keys)) { - yield { cid: new CID(cid), type: pin.Keys[cid].Type } + yield toPin(pin.Keys[cid].Type, cid, pin.Keys[cid].Comments) } return } - yield { cid: new CID(pin.Cid), type: pin.Type } + yield toPin(pin.Type, pin.Cid, pin.Comments) } } }) diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 333633a58b..aa2ebf24ed 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -184,7 +184,7 @@ "go-ipfs-dep": "0.4.23-3", "hat": "0.0.3", "interface-ipfs-core": "^0.132.0", - "ipfs-interop": "^1.0.0", + "ipfs-interop": "github:ipfs/interop#refactor/streaming-pin-api", "ipfsd-ctl": "^3.0.0", "it-first": "^1.0.1", "ncp": "^2.0.0", diff --git a/packages/ipfs/src/core/components/init.js b/packages/ipfs/src/core/components/init.js index 2fda6c3490..6d82c358ae 100644 --- a/packages/ipfs/src/core/components/init.js +++ b/packages/ipfs/src/core/components/init.js @@ -28,6 +28,7 @@ const OfflineDatastore = require('../ipns/routing/offline-datastore') const initAssets = require('../runtime/init-assets-nodejs') const PinManager = require('./pin/pin-manager') const Components = require('./') +const drain = require('it-drain') module.exports = ({ apiManager, @@ -124,7 +125,7 @@ module.exports = ({ if (!isInitialized && !options.emptyRepo) { // add empty unixfs dir object (go-ipfs assumes this exists) - const emptyDirCid = await addEmptyDir({ dag }) + const emptyDirCid = await addEmptyDir({ dag, pin }) log('adding default assets') await initAssets({ add, print }) @@ -259,14 +260,17 @@ function createPeerId ({ privateKey, bits, print }) { } } -function addEmptyDir ({ dag }) { +async function addEmptyDir ({ dag, pin }) { const node = new DAGNode(new UnixFs('directory').marshal()) - return dag.put(node, { + const cid = await dag.put(node, { version: 0, format: multicodec.DAG_PB, hashAlg: multicodec.SHA2_256, preload: false }) + await drain(pin.add(cid)) + + return cid } // Apply profiles (e.g. ['server', 'lowpower']) to config diff --git a/packages/ipfs/src/core/components/pin/ls.js b/packages/ipfs/src/core/components/pin/ls.js index bfdff150e7..86e07c318c 100644 --- a/packages/ipfs/src/core/components/pin/ls.js +++ b/packages/ipfs/src/core/components/pin/ls.js @@ -5,14 +5,14 @@ const { resolvePath } = require('../../utils') const PinManager = require('./pin-manager') const { PinTypes } = PinManager -function toPin (type, cid, name) { +function toPin (type, cid, comments) { const output = { type, cid } - if (name) { - output.name = name + if (comments) { + output.comments = comments } return output @@ -80,8 +80,8 @@ module.exports = ({ pinManager, dag }) => { } if (type === PinTypes.recursive || type === PinTypes.all) { - for await (const { cid, name } of pinManager.recursiveKeys()) { - yield toPin(PinTypes.recursive, cid, name) + for await (const { cid, comments } of pinManager.recursiveKeys()) { + yield toPin(PinTypes.recursive, cid, comments) } } @@ -95,8 +95,8 @@ module.exports = ({ pinManager, dag }) => { } if (type === PinTypes.direct || type === PinTypes.all) { - for await (const { cid, name } of pinManager.directKeys()) { - yield toPin(PinTypes.direct, cid, name) + for await (const { cid, comments } of pinManager.directKeys()) { + yield toPin(PinTypes.direct, cid, comments) } } } diff --git a/packages/ipfs/src/core/components/pin/pin-manager.js b/packages/ipfs/src/core/components/pin/pin-manager.js index 24ea7c480d..5669d4c6f7 100644 --- a/packages/ipfs/src/core/components/pin/pin-manager.js +++ b/packages/ipfs/src/core/components/pin/pin-manager.js @@ -62,30 +62,11 @@ class PinManager { return this.repo.pins.put(toKey(cid), cbor.encode({ cid: cid.buffer, type: PinTypes.direct, - name: options.name + comments: options.comments })) } - async unpin (cid) { - if (typeof cid === 'string' || cid instanceof String) { - // find pin with passed name - const result = await first(this.repo.pins.query({ - filters: [entry => { - const pin = cbor.decode(entry.value) - - return pin.name === cid - }], - limit: 1 - })) - - if (!result) { - // no pin with this name - return - } - - cid = new CID(cbor.decode(result.value).cid) - } - + async unpin (cid) { // eslint-disable-line require-await return this.repo.pins.delete(toKey(cid)) } @@ -95,7 +76,7 @@ class PinManager { await this.repo.pins.put(toKey(cid), cbor.encode({ cid: cid.buffer, type: PinTypes.recursive, - name: options.name + comments: options.comments })) } @@ -111,7 +92,7 @@ class PinManager { yield { cid: new CID(pin.cid), - name: pin.name + comments: pin.comments } } } @@ -128,7 +109,7 @@ class PinManager { yield { cid: new CID(pin.cid), - name: pin.name + comments: pin.comments } } } diff --git a/packages/ipfs/src/http/api/resources/pin.js b/packages/ipfs/src/http/api/resources/pin.js index 4d2e860307..85231acd64 100644 --- a/packages/ipfs/src/http/api/resources/pin.js +++ b/packages/ipfs/src/http/api/resources/pin.js @@ -62,8 +62,8 @@ exports.ls = { if (!request.query.stream) { const res = await pipe( ipfs.pin.ls(path, { type }), - reduce((res, { type, cid }) => { - res.Keys[cidToString(cid, { base: request.query['cid-base'] })] = { Type: type } + reduce((res, { type, cid, comments }) => { + res.Keys[cidToString(cid, { base: request.query['cid-base'] })] = { Type: type, Comments: comments } return res }, { Keys: {} }) ) @@ -73,7 +73,7 @@ exports.ls = { return streamResponse(request, h, () => pipe( ipfs.pin.ls(path, { type }), - map(({ type, cid }) => ({ Type: type, Cid: cidToString(cid, { base: request.query['cid-base'] }) })), + map(({ type, cid, comments }) => ({ Type: type, Cid: cidToString(cid, { base: request.query['cid-base'] }), Comments: comments })), ndjson.stringify )) } From 6ff3a10e9c4049a9228cc7ade180a0f490f77145 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 5 Mar 2020 18:15:16 +0000 Subject: [PATCH 04/40] chore: dedupe pinning tests --- packages/interface-ipfs-core/src/pin/add.js | 136 +++++- packages/interface-ipfs-core/src/pin/index.js | 5 +- packages/interface-ipfs-core/src/pin/ls.js | 14 + packages/interface-ipfs-core/src/pin/pins.js | 460 ------------------ packages/interface-ipfs-core/src/pin/rm.js | 56 ++- packages/interface-ipfs-core/src/pin/utils.js | 50 +- 6 files changed, 244 insertions(+), 477 deletions(-) delete mode 100644 packages/interface-ipfs-core/src/pin/pins.js diff --git a/packages/interface-ipfs-core/src/pin/add.js b/packages/interface-ipfs-core/src/pin/add.js index 6c2e3559a4..e755bf411b 100644 --- a/packages/interface-ipfs-core/src/pin/add.js +++ b/packages/interface-ipfs-core/src/pin/add.js @@ -1,9 +1,13 @@ /* eslint-env mocha */ 'use strict' -const { fixtures } = require('./utils') +const { fixtures, clearPins, expectPinned, pinTypes } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const all = require('it-all') +const drain = require('it-drain') +const { + DAGNode +} = require('ipld-dag-pb') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -20,16 +24,144 @@ module.exports = (common, options) => { let ipfs before(async () => { ipfs = (await common.spawn()).api + await Promise.all(fixtures.files.map(file => { return all(ipfs.add(file.data, { pin: false })) })) + + await all( + ipfs.add(fixtures.directory.files.map( + file => ({ + path: file.path, + content: file.data + }) + ), { + pin: false + }) + ) }) after(() => common.clean()) - it('should add a pin', async () => { + beforeEach(() => { + return clearPins(ipfs) + }) + + it('should add a pin and return the added CID', async () => { const pinset = await all(ipfs.pin.add(fixtures.files[0].cid, { recursive: false })) expect(pinset.map(p => p.cid)).to.deep.include(fixtures.files[0].cid) }) + + it('should add recursively', async () => { + await drain(ipfs.pin.add(fixtures.directory.cid)) + await expectPinned(ipfs, fixtures.directory.cid, pinTypes.recursive) + + const pinChecks = Object.values(fixtures.directory.files).map(file => expectPinned(ipfs, file.cid)) + return Promise.all(pinChecks) + }) + + it('should add directly', async () => { + await drain(ipfs.pin.add(fixtures.directory.cid, { recursive: false })) + await Promise.all([ + expectPinned(ipfs, fixtures.directory.cid, pinTypes.direct), + expectPinned(ipfs, fixtures.directory.files[0].cid, false) + ]) + }) + + it('should recursively pin parent of direct pin', async () => { + await drain(ipfs.pin.add(fixtures.directory.files[0].cid, { recursive: false })) + await drain(ipfs.pin.add(fixtures.directory.cid)) + await Promise.all([ + // file is pinned both directly and indirectly o.O + expectPinned(ipfs, fixtures.directory.files[0].cid, pinTypes.direct), + expectPinned(ipfs, fixtures.directory.files[0].cid, pinTypes.indirect) + ]) + }) + + it('should fail to directly pin a recursive pin', async () => { + await drain(ipfs.pin.add(fixtures.directory.cid)) + return expect(drain(ipfs.pin.add(fixtures.directory.cid, { recursive: false }))) + .to.eventually.be.rejected() + .with(/already pinned recursively/) + }) + + it('should fail to pin a hash not in datastore', function () { + this.slow(3 * 1000) + this.timeout(5 * 1000) + const falseHash = `${`${fixtures.directory.cid}`.slice(0, -2)}ss` + return expect(drain(ipfs.pin.add(falseHash, { timeout: '2s' }))) + .to.eventually.be.rejected() + // TODO: http api TimeoutErrors do not have this property + // .with.a.property('code').that.equals('ERR_TIMEOUT') + }) + + it('needs all children in datastore to pin recursively', async function () { + this.slow(3 * 1000) + this.timeout(5 * 1000) + await all(ipfs.block.rm(fixtures.directory.files[0].cid)) + + await expect(drain(ipfs.pin.add(fixtures.directory.cid, { timeout: '2s' }))) + .to.eventually.be.rejected() + }) + + it('should pin dag-cbor', async () => { + const cid = await ipfs.dag.put({}, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + await drain(ipfs.pin.add(cid)) + + const pins = await all(ipfs.pin.ls()) + + expect(pins).to.deep.include({ + type: 'recursive', + cid + }) + }) + + it('should pin raw', async () => { + const cid = await ipfs.dag.put(Buffer.alloc(0), { + format: 'raw', + hashAlg: 'sha2-256' + }) + + await drain(ipfs.pin.add(cid)) + + const pins = await all(ipfs.pin.ls()) + + expect(pins).to.deep.include({ + type: 'recursive', + cid + }) + }) + + it('should pin dag-cbor with dag-pb child', async () => { + const child = await ipfs.dag.put(new DAGNode(Buffer.from(`${Math.random()}`)), { + format: 'dag-pb', + hashAlg: 'sha2-256' + }) + const parent = await ipfs.dag.put({ + child + }, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + await drain(ipfs.pin.add(parent, { + recursive: true + })) + + const pins = await all(ipfs.pin.ls()) + + expect(pins).to.deep.include({ + cid: parent, + type: 'recursive' + }) + expect(pins).to.deep.include({ + cid: child, + type: 'indirect' + }) + }) }) } diff --git a/packages/interface-ipfs-core/src/pin/index.js b/packages/interface-ipfs-core/src/pin/index.js index ad97a96df0..15f55419d3 100644 --- a/packages/interface-ipfs-core/src/pin/index.js +++ b/packages/interface-ipfs-core/src/pin/index.js @@ -2,10 +2,9 @@ const { createSuite } = require('../utils/suite') const tests = { - ls: require('./ls'), - rm: require('./rm'), add: require('./add'), - core: require('./pins') + ls: require('./ls'), + rm: require('./rm') } module.exports = createSuite(tests) diff --git a/packages/interface-ipfs-core/src/pin/ls.js b/packages/interface-ipfs-core/src/pin/ls.js index 29d21f2e9a..9345eabe10 100644 --- a/packages/interface-ipfs-core/src/pin/ls.js +++ b/packages/interface-ipfs-core/src/pin/ls.js @@ -159,5 +159,19 @@ module.exports = (common, options) => { expect(cids).to.deep.include(fixtures.files[0].cid) expect(cids).to.deep.include(fixtures.files[1].cid) }) + + it('should throw error for invalid non-string pin type option', () => { + return expect(all(ipfs.pin.ls({ type: 6 }))) + .to.eventually.be.rejected() + // TODO: go-ipfs does not return error codes + // .with.property('code').that.equals('ERR_INVALID_PIN_TYPE') + }) + + it('should throw error for invalid string pin type option', () => { + return expect(all(ipfs.pin.ls({ type: '__proto__' }))) + .to.eventually.be.rejected() + // TODO: go-ipfs does not return error codes + // .with.property('code').that.equals('ERR_INVALID_PIN_TYPE') + }) }) } diff --git a/packages/interface-ipfs-core/src/pin/pins.js b/packages/interface-ipfs-core/src/pin/pins.js deleted file mode 100644 index a171d0b91c..0000000000 --- a/packages/interface-ipfs-core/src/pin/pins.js +++ /dev/null @@ -1,460 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const { getDescribe, getIt, expect } = require('../utils/mocha') -const { - DAGNode -} = require('ipld-dag-pb') -const all = require('it-all') -const last = require('it-last') -const drain = require('it-drain') -const CID = require('cids') - -// fixture structure: -// planets/ -// solar-system.md -// mercury/ -// wiki.md -/* const pins = { - root: new CID('QmTAMavb995EHErSrKo7mB8dYkpaSJxu6ys1a6XJyB2sys'), - solarWiki: new CID('QmTMbkDfvHwq3Aup6Nxqn3KKw9YnoKzcZvuArAfQ9GF3QG'), - mercuryDir: new CID('QmbJCNKXJqVK8CzbjpNFz2YekHwh3CSHpBA86uqYg3sJ8q'), - mercuryWiki: new CID('QmVgSHAdMxFAuMP2JiMAYkB8pCWP1tcB9djqvq8GKAFiHi') -} */ -let pins -const pinTypes = { - direct: 'direct', - recursive: 'recursive', - indirect: 'indirect', - all: 'all' -} - -/** @typedef { import("ipfsd-ctl/src/factory") } Factory */ -/** - * @param {Factory} common - * @param {Object} options - */ -module.exports = (common, options) => { - const describe = getDescribe(options) - const it = getIt(options) - - describe('pin', function () { - this.timeout(50 * 1000) - - const fixtures = [{ - path: 'planets/mercury/wiki.md', - content: 'solar system content' - }, { - path: 'planets/solar-system.md', - content: 'wiki content' - }] - - let ipfs - - async function isPinnedWithType (path, type) { - try { - for await (const _ of ipfs.pin.ls(path, { type })) { // eslint-disable-line no-unused-vars - return true - } - return false - } catch (err) { - return false - } - } - - async function expectPinned (cid, type = pinTypes.all, pinned = true) { - if (typeof type === 'boolean') { - pinned = type - type = pinTypes.all - } - - const result = await isPinnedWithType(cid, type) - expect(result).to.eql(pinned) - } - - async function clearPins () { - for await (const { cid } of ipfs.pin.ls({ type: pinTypes.recursive })) { - await drain(ipfs.pin.rm(cid)) - } - - for await (const { cid } of ipfs.pin.ls({ type: pinTypes.direct })) { - await drain(ipfs.pin.rm(cid)) - } - } - - before(async () => { - ipfs = (await common.spawn()).api - const added = (await all(ipfs.add(fixtures))).reduce((acc, curr) => { - acc[curr.path] = curr.cid - - return acc - }, {}) - - pins = { - root: added.planets, - solarWiki: added['planets/solar-system.md'], - mercuryDir: added['planets/mercury'], - mercuryWiki: added['planets/mercury/wiki.md'] - } - }) - - after(() => common.clean()) - - describe('pinned status', function () { - beforeEach(async () => { - await clearPins() - await drain(ipfs.pin.add(pins.root)) - }) - - it('should be pinned when added', async () => { - await drain(ipfs.pin.add(pins.solarWiki)) - return expectPinned(pins.solarWiki) - }) - - it('should not be pinned when not in datastore', () => { - const falseHash = `${`${pins.root}`.slice(0, -2)}ss` - return expectPinned(falseHash, false) - }) - - it('should not be pinned when in datastore but not added', async () => { - await drain(ipfs.pin.rm(pins.root)) - return expectPinned(pins.root, false) - }) - - it('should be pinned recursively when added', () => { - return expectPinned(pins.root, pinTypes.recursive) - }) - - it('should be pinned indirectly', () => { - return expectPinned(pins.mercuryWiki, pinTypes.indirect) - }) - - it('should be pinned directly', async () => { - await drain(ipfs.pin.add(pins.mercuryDir, { recursive: false })) - return expectPinned(pins.mercuryDir, pinTypes.direct) - }) - - it('should not be pinned when not in datastore or added', async () => { - await clearPins() - return expectPinned(pins.mercuryDir, pinTypes.direct, false) - }) - }) - - describe('add', function () { - beforeEach(function () { - return clearPins() - }) - - it('should add recursively', async () => { - await drain(ipfs.pin.add(pins.root)) - await expectPinned(pins.root, pinTypes.recursive) - - const pinChecks = Object.values(pins).map(hash => expectPinned(hash)) - return Promise.all(pinChecks) - }) - - it('should add directly', async () => { - await drain(ipfs.pin.add(pins.root, { recursive: false })) - await Promise.all([ - expectPinned(pins.root, pinTypes.direct), - expectPinned(pins.solarWiki, false) - ]) - }) - - it('should recursively pin parent of direct pin', async () => { - await drain(ipfs.pin.add(pins.solarWiki, { recursive: false })) - await drain(ipfs.pin.add(pins.root)) - await Promise.all([ - // solarWiki is pinned both directly and indirectly o.O - expectPinned(pins.solarWiki, pinTypes.direct), - expectPinned(pins.solarWiki, pinTypes.indirect) - ]) - }) - - it('should fail to directly pin a recursive pin', async () => { - await drain(ipfs.pin.add(pins.root)) - return expect(last(ipfs.pin.add(pins.root, { recursive: false }))) - .to.eventually.be.rejected() - .with(/already pinned recursively/) - }) - - it('should fail to pin a hash not in datastore', function () { - this.timeout(5 * 1000) - const falseHash = `${`${pins.root}`.slice(0, -2)}ss` - return expect(last(ipfs.pin.add(falseHash, { timeout: '2s' }))) - .to.eventually.be.rejected() - // TODO: http api TimeoutErrors do not have this property - // .with.a.property('code').that.equals('ERR_TIMEOUT') - }) - - // TODO block rm breaks subsequent tests - // it.skip('needs all children in datastore to pin recursively', () => { - // return ipfs.block.rm(pins.mercuryWiki) - // .then(() => expectTimeout(pin.add(pins.root), 4000)) - // }) - }) - - describe('ls', function () { - before(async () => { - await clearPins() - await drain(ipfs.pin.add(pins.root)) - await drain(ipfs.pin.add(pins.mercuryDir, { recursive: false })) - }) - - it('should list pins of a particular CID', async () => { - const out = await all(ipfs.pin.ls(pins.mercuryDir)) - expect(out[0].cid).to.deep.equal(pins.mercuryDir) - expect(out[0].type).to.eql(pinTypes.direct) - }) - - it('should list indirect pins that supersede direct pins', async () => { - const ls = await all(ipfs.pin.ls()) - - const pinType = ls.find(out => out.cid.equals(pins.mercuryDir)).type - expect(pinType).to.eql(pinTypes.indirect) - }) - - it('should list all pins', async () => { - const out = await all(ipfs.pin.ls()) - - expect(out).to.deep.include.members([ - { - type: 'recursive', - cid: new CID(pins.root) - }, - { - type: 'indirect', - cid: new CID(pins.solarWiki) - }, - { - type: 'indirect', - cid: new CID(pins.mercuryDir) - }, - { - type: 'indirect', - cid: new CID(pins.mercuryWiki) - } - ]) - }) - - it('should list all direct pins', async () => { - const out = await all(ipfs.pin.ls({ type: 'direct' })) - - expect(out).to.deep.include.members([ - { - type: 'direct', - cid: new CID(pins.mercuryDir) - } - ]) - }) - - it('should list all recursive pins', async () => { - const out = await all(ipfs.pin.ls({ type: 'recursive' })) - - expect(out).to.deep.include.members([ - { - type: 'recursive', - cid: new CID(pins.root) - } - ]) - }) - - it('should list all indirect pins', async () => { - const out = await all(ipfs.pin.ls({ type: 'indirect' })) - - expect(out).to.deep.include.members([ - { - type: 'indirect', - cid: new CID(pins.solarWiki) - }, - { - type: 'indirect', - cid: new CID(pins.mercuryDir) - }, - { - type: 'indirect', - cid: new CID(pins.mercuryWiki) - } - ]) - }) - - it('should list direct pins for CID', async () => { - const out = await all(ipfs.pin.ls(pins.mercuryDir, { type: 'direct' })) - - expect(out).to.have.deep.members([ - { - type: 'direct', - cid: new CID(pins.mercuryDir) - } - ]) - }) - - it('should list direct pins for path', async () => { - const out = await all(ipfs.pin.ls(`/ipfs/${pins.root}/mercury/`, { type: 'direct' })) - - expect(out).to.have.deep.members([ - { - type: 'direct', - cid: new CID(pins.mercuryDir) - } - ]) - }) - - it('should list direct pins for path (no match)', () => { - return expect(all(ipfs.pin.ls(`/ipfs/${pins.root}/mercury/wiki.md`, { type: 'direct' }))) - .to.eventually.be.rejected() - }) - - it('should list direct pins for CID (no match)', () => { - return expect(all(ipfs.pin.ls(pins.root, { type: 'direct' }))) - .to.eventually.be.rejected() - }) - - it('should list recursive pins for CID', async () => { - const out = await all(ipfs.pin.ls(pins.root, { type: 'recursive' })) - - expect(out).to.have.deep.members([ - { - type: 'recursive', - cid: new CID(pins.root) - } - ]) - }) - - it('should list recursive pins for CID (no match)', () => { - return expect(all(ipfs.pin.ls(pins.mercuryDir, { type: 'recursive' }))) - .to.eventually.be.rejected() - }) - - it('should list indirect pins for CID', async () => { - const out = await all(ipfs.pin.ls(pins.solarWiki, { type: 'indirect' })) - - expect(out).to.have.deep.members([ - { - type: `indirect through ${pins.root}`, - cid: new CID(pins.solarWiki) - } - ]) - }) - - it('should list indirect pins for CID (no match)', () => { - return expect(all(ipfs.pin.ls(pins.root, { type: 'indirect' }))) - .to.eventually.be.rejected() - }) - }) - - describe('rm', function () { - beforeEach(async () => { - await clearPins() - await drain(ipfs.pin.add(pins.root)) - }) - - it('should remove a recursive pin', async () => { - await drain(ipfs.pin.rm(pins.root)) - await Promise.all([ - expectPinned(pins.root, false), - expectPinned(pins.mercuryWiki, false) - ]) - }) - - it('should remove a direct pin', async () => { - await clearPins() - await drain(ipfs.pin.add(pins.mercuryDir, { recursive: false })) - await drain(ipfs.pin.rm(pins.mercuryDir)) - await expectPinned(pins.mercuryDir, false) - }) - - it('should fail to remove an indirect pin', async () => { - await expect(last(ipfs.pin.rm(pins.solarWiki))) - .to.eventually.be.rejected() - .with(/is pinned indirectly under/) - await expectPinned(pins.solarWiki) - }) - - it('should fail when an item is not pinned', async () => { - await drain(ipfs.pin.rm(pins.root)) - await expect(last(ipfs.pin.rm(pins.root))) - .to.eventually.be.rejected() - .with(/is not pinned/) - }) - }) - - describe('non-dag-pb nodes', function () { - it('should pin dag-cbor', async () => { - const cid = await ipfs.dag.put({}, { - format: 'dag-cbor', - hashAlg: 'sha2-256' - }) - - await drain(ipfs.pin.add(cid)) - - const pins = await all(ipfs.pin.ls()) - - expect(pins).to.deep.include({ - type: 'recursive', - cid - }) - }) - - it('should pin raw', async () => { - const cid = await ipfs.dag.put(Buffer.alloc(0), { - format: 'raw', - hashAlg: 'sha2-256' - }) - - await drain(ipfs.pin.add(cid)) - - const pins = await all(ipfs.pin.ls()) - - expect(pins).to.deep.include({ - type: 'recursive', - cid - }) - }) - - it('should pin dag-cbor with dag-pb child', async () => { - const child = await ipfs.dag.put(new DAGNode(Buffer.from(`${Math.random()}`)), { - format: 'dag-pb', - hashAlg: 'sha2-256' - }) - const parent = await ipfs.dag.put({ - child - }, { - format: 'dag-cbor', - hashAlg: 'sha2-256' - }) - - await drain(ipfs.pin.add(parent, { - recursive: true - })) - - const pins = await all(ipfs.pin.ls()) - - expect(pins).to.deep.include({ - cid: parent, - type: 'recursive' - }) - expect(pins).to.deep.include({ - cid: child, - type: 'indirect' - }) - }) - }) - - describe('ls', () => { - it('should throw error for invalid non-string pin type option', () => { - return expect(all(ipfs.pin.ls({ type: 6 }))) - .to.eventually.be.rejected() - // TODO: go-ipfs does not return error codes - // .with.property('code').that.equals('ERR_INVALID_PIN_TYPE') - }) - - it('should throw error for invalid string pin type option', () => { - return expect(all(ipfs.pin.ls({ type: '__proto__' }))) - .to.eventually.be.rejected() - // TODO: go-ipfs does not return error codes - // .with.property('code').that.equals('ERR_INVALID_PIN_TYPE') - }) - }) - }) -} diff --git a/packages/interface-ipfs-core/src/pin/rm.js b/packages/interface-ipfs-core/src/pin/rm.js index 91bc802ea5..e266af8995 100644 --- a/packages/interface-ipfs-core/src/pin/rm.js +++ b/packages/interface-ipfs-core/src/pin/rm.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { fixtures } = require('./utils') +const { fixtures, expectPinned, clearPins } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const all = require('it-all') const drain = require('it-drain') @@ -21,31 +21,65 @@ module.exports = (common, options) => { let ipfs before(async () => { ipfs = (await common.spawn()).api - await drain(ipfs.add(fixtures.files[0].data, { pin: false })) - await drain(ipfs.pin.add(fixtures.files[0].cid, { recursive: true })) - await drain(ipfs.add(fixtures.files[1].data, { pin: false })) - await drain(ipfs.pin.add(fixtures.files[1].cid, { recursive: false })) + + await Promise.all(fixtures.files.map(file => { + return all(ipfs.add(file.data, { pin: false })) + })) + + await all( + ipfs.add(fixtures.directory.files.map( + file => ({ + path: file.path, + content: file.data + }) + ), { + pin: false + }) + ) }) after(() => common.clean()) + beforeEach(() => { + return clearPins(ipfs) + }) + it('should remove a recursive pin', async () => { - const removedPinset = await all(ipfs.pin.rm(fixtures.files[0].cid, { recursive: true })) - expect(removedPinset.map(p => p.cid)).to.deep.equal([fixtures.files[0].cid]) + await drain(ipfs.pin.add(fixtures.directory.cid)) + + const removedPinset = await all(ipfs.pin.rm(fixtures.directory.cid, { recursive: true })) + expect(removedPinset.map(p => p.cid)).to.deep.equal([fixtures.directory.cid]) const pinset = await all(ipfs.pin.ls({ type: 'recursive' })) expect(pinset).to.not.deep.include({ - cid: fixtures.files[0].cid, + cid: fixtures.directory.cid, type: 'recursive' }) }) it('should remove a direct pin', async () => { - const removedPinset = await all(ipfs.pin.rm(fixtures.files[1].cid, { recursive: false })) - expect(removedPinset.map(p => p.cid)).to.deep.equal([fixtures.files[1].cid]) + await drain(ipfs.pin.add(fixtures.directory.cid, { recursive: false })) + + const removedPinset = await all(ipfs.pin.rm(fixtures.directory.cid, { recursive: false })) + expect(removedPinset.map(p => p.cid)).to.deep.equal([fixtures.directory.cid]) const pinset = await all(ipfs.pin.ls({ type: 'direct' })) - expect(pinset.map(p => p.cid)).to.not.deep.include(fixtures.files[1].cid) + expect(pinset.map(p => p.cid)).to.not.deep.include(fixtures.directory.cid) + }) + + it('should fail to remove an indirect pin', async () => { + await drain(ipfs.pin.add(fixtures.directory.cid)) + + await expect(drain(ipfs.pin.rm(fixtures.directory.files[0].cid))) + .to.eventually.be.rejected() + .with(/is pinned indirectly under/) + await expectPinned(ipfs, fixtures.directory.files[0].cid) + }) + + it('should fail when an item is not pinned', async () => { + await expect(drain(ipfs.pin.rm(fixtures.directory.cid))) + .to.eventually.be.rejected() + .with(/is not pinned/) }) }) } diff --git a/packages/interface-ipfs-core/src/pin/utils.js b/packages/interface-ipfs-core/src/pin/utils.js index 34b33eee6b..dcb09c633a 100644 --- a/packages/interface-ipfs-core/src/pin/utils.js +++ b/packages/interface-ipfs-core/src/pin/utils.js @@ -1,9 +1,18 @@ 'use strict' +const { expect } = require('../utils/mocha') const loadFixture = require('aegir/fixtures') const CID = require('cids') +const drain = require('it-drain') -exports.fixtures = Object.freeze({ +const pinTypes = { + direct: 'direct', + recursive: 'recursive', + indirect: 'indirect', + all: 'all' +} + +const fixtures = Object.freeze({ // NOTE: files under 'directory' need to be different than standalone ones in 'files' directory: Object.freeze({ cid: new CID('QmY8KdYQSYKFU5hM7F5ioZ5yYSgV5VZ1kDEdqfRL3rFgcd'), @@ -25,3 +34,42 @@ exports.fixtures = Object.freeze({ cid: new CID('QmY9cxiHqTFoWamkQVkpmmqzBrY3hCBEL2XNu3NtX74Fuu') })]) }) + +const clearPins = async (ipfs) => { + for await (const { cid } of ipfs.pin.ls({ type: pinTypes.recursive })) { + await drain(ipfs.pin.rm(cid)) + } + + for await (const { cid } of ipfs.pin.ls({ type: pinTypes.direct })) { + await drain(ipfs.pin.rm(cid)) + } +} + +const expectPinned = async (ipfs, cid, type = pinTypes.all, pinned = true) => { + if (typeof type === 'boolean') { + pinned = type + type = pinTypes.all + } + + const result = await isPinnedWithType(ipfs, cid, type) + expect(result).to.eql(pinned) +} + +async function isPinnedWithType (ipfs, path, type) { + try { + for await (const _ of ipfs.pin.ls(path, { type })) { // eslint-disable-line no-unused-vars + return true + } + return false + } catch (err) { + return false + } +} + +module.exports = { + fixtures, + clearPins, + expectPinned, + isPinnedWithType, + pinTypes +} From e9ab2335b46a36647305c31070116406a435136b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 5 Mar 2020 18:19:27 +0000 Subject: [PATCH 05/40] chore: fix up http tests --- packages/ipfs/src/http/api/resources/pin.js | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/ipfs/src/http/api/resources/pin.js b/packages/ipfs/src/http/api/resources/pin.js index 85231acd64..65c32e213e 100644 --- a/packages/ipfs/src/http/api/resources/pin.js +++ b/packages/ipfs/src/http/api/resources/pin.js @@ -11,6 +11,22 @@ const { cidToString } = require('../../../utils/cid') const streamResponse = require('../../utils/stream-response') const all = require('it-all') +function toPin (type, cid, comments) { + const output = { + Type: type + } + + if (cid) { + output.Cid = cid + } + + if (comments) { + output.Comments = comments + } + + return output +} + function parseArgs (request, h) { let { arg } = request.query @@ -63,7 +79,7 @@ exports.ls = { const res = await pipe( ipfs.pin.ls(path, { type }), reduce((res, { type, cid, comments }) => { - res.Keys[cidToString(cid, { base: request.query['cid-base'] })] = { Type: type, Comments: comments } + res.Keys[cidToString(cid, { base: request.query['cid-base'] })] = toPin(type, null, comments) return res }, { Keys: {} }) ) @@ -73,7 +89,7 @@ exports.ls = { return streamResponse(request, h, () => pipe( ipfs.pin.ls(path, { type }), - map(({ type, cid, comments }) => ({ Type: type, Cid: cidToString(cid, { base: request.query['cid-base'] }), Comments: comments })), + map(({ type, cid, comments }) => toPin(type, cidToString(cid, { base: request.query['cid-base'] }), comments)), ndjson.stringify )) } From 68345b28504969b1f2a0fe2c130f9da423e857af Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 6 Mar 2020 15:54:49 +0000 Subject: [PATCH 06/40] feat: new api --- packages/interface-ipfs-core/SPEC/PIN.md | 8 +- packages/interface-ipfs-core/src/pin/add.js | 96 +++++++++++++- packages/interface-ipfs-core/src/pin/ls.js | 36 ++++- packages/interface-ipfs-core/src/pin/rm.js | 8 ++ packages/interface-ipfs-core/src/pin/utils.js | 4 +- packages/ipfs-http-client/src/pin/add.js | 28 ++-- packages/ipfs-http-client/src/pin/rm.js | 27 ++-- .../ipfs-http-client/test/interface.spec.js | 9 +- packages/ipfs-utils/package.json | 1 + .../ipfs-utils/src/pins/normalise-input.js | 124 ++++++++++++++++++ .../test/pins/normalise-input.spec.js | 70 ++++++++++ packages/ipfs/src/cli/commands/pin/add.js | 11 +- packages/ipfs/src/cli/commands/pin/ls.js | 4 + packages/ipfs/src/cli/commands/pin/rm.js | 4 +- packages/ipfs/src/core/components/pin/add.js | 18 +-- packages/ipfs/src/core/components/pin/ls.js | 38 ++---- .../src/core/components/pin/pin-manager.js | 3 +- packages/ipfs/src/core/components/pin/rm.js | 11 +- packages/ipfs/src/core/utils.js | 61 ++++----- packages/ipfs/src/http/api/resources/pin.js | 45 +++---- packages/ipfs/test/cli/pin.js | 34 +++-- 21 files changed, 487 insertions(+), 153 deletions(-) create mode 100644 packages/ipfs-utils/src/pins/normalise-input.js create mode 100644 packages/ipfs-utils/test/pins/normalise-input.spec.js diff --git a/packages/interface-ipfs-core/SPEC/PIN.md b/packages/interface-ipfs-core/SPEC/PIN.md index 20c2d4e696..45279f21d5 100644 --- a/packages/interface-ipfs-core/SPEC/PIN.md +++ b/packages/interface-ipfs-core/SPEC/PIN.md @@ -12,9 +12,8 @@ Where: -- `source` is a [CID], an array of CIDs or an (async) iterable that yields CIDs +- `source` is a [CID], an IPFS path or an object of the form `{ cid: CID, recursive: Boolean, comments: String }` or `{ path: String, recursive: Boolean, comments: String }` or an array or (async) iterable that yields any of the previous types - `options` is an object that can contain the following keys - - `recursive` (`boolean`) - Recursively pin the object linked. Type: bool. Default: `true` - `timeout` (`number`|`string`) - Throw an error if the request does not complete within the specified milliseconds timeout. If `timeout` is a string, the value is parsed as a [human readable duration](https://www.npmjs.com/package/parse-duration). There is no timeout by default. **Returns** @@ -34,7 +33,7 @@ Each yielded object has the form: **Example:** ```JavaScript -for await (const pin of ipfs.pin.add('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')) { +for await (const pin of ipfs.pin.add(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) { console.log(pinset) } // { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') } @@ -80,9 +79,8 @@ A great source of [examples][] can be found in the tests for this API. ##### `ipfs.pin.rm(source, [options])` Where: -- `source` is a [CID], an array of CIDs or an (async) iterable that yields CIDs +- `source` is a [CID], an IPFS path or an object of the form `{ cid: CID, recursive: Boolean}` or `{ path: String, recursive: Boolean }` or an array or (async) iterable that yields any of the previous types - `options` is an object that can contain the following keys - - 'recursive' - Recursively unpin the object linked. Type: bool. Default: `true` **Returns** diff --git a/packages/interface-ipfs-core/src/pin/add.js b/packages/interface-ipfs-core/src/pin/add.js index e755bf411b..c8893388c7 100644 --- a/packages/interface-ipfs-core/src/pin/add.js +++ b/packages/interface-ipfs-core/src/pin/add.js @@ -47,11 +47,88 @@ module.exports = (common, options) => { return clearPins(ipfs) }) - it('should add a pin and return the added CID', async () => { - const pinset = await all(ipfs.pin.add(fixtures.files[0].cid, { recursive: false })) + async function testAddInput (source) { + const pinset = await all(ipfs.pin.add(source)) + + expect(pinset.map(p => p.cid)).to.have.deep.members([ + fixtures.files[0].cid, + fixtures.files[1].cid + ]) + } + + it('should add a CID and return the added CID', async () => { + const pinset = await all(ipfs.pin.add(fixtures.files[0].cid)) + expect(pinset.map(p => p.cid)).to.deep.include(fixtures.files[0].cid) + }) + + it('should add a pin with options and return the added CID', async () => { + const pinset = await all(ipfs.pin.add({ + cid: fixtures.files[0].cid, + recursive: false + })) expect(pinset.map(p => p.cid)).to.deep.include(fixtures.files[0].cid) }) + it('should add an array of CIDs', () => { + return testAddInput([ + fixtures.files[0].cid, + fixtures.files[1].cid + ]) + }) + + it('should add a generator of CIDs', () => { + return testAddInput(function * () { + yield fixtures.files[0].cid + yield fixtures.files[1].cid + }()) + }) + + it('should add an async generator of CIDs', () => { + return testAddInput(async function * () { // eslint-disable-line require-await + yield fixtures.files[0].cid + yield fixtures.files[1].cid + }()) + }) + + it('should add an array of pins with options', () => { + return testAddInput([ + { + cid: fixtures.files[0].cid, + recursive: false + }, + { + cid: fixtures.files[1].cid, + recursive: true + } + ]) + }) + + it('should add a generator of pins with options', () => { + return testAddInput(function * () { + yield { + cid: fixtures.files[0].cid, + recursive: false + } + yield { + cid: fixtures.files[1].cid, + recursive: true + } + }()) + }) + + it('should add an async generator of pins with options', () => { + return testAddInput(async function * () { // eslint-disable-line require-await + yield { + cid: fixtures.files[0].cid, + recursive: false + } + yield { + cid: fixtures.files[1].cid, + recursive: true + } + }()) + }) + it('should add recursively', async () => { await drain(ipfs.pin.add(fixtures.directory.cid)) await expectPinned(ipfs, fixtures.directory.cid, pinTypes.recursive) @@ -61,7 +138,10 @@ module.exports = (common, options) => { }) it('should add directly', async () => { - await drain(ipfs.pin.add(fixtures.directory.cid, { recursive: false })) + await drain(ipfs.pin.add({ + cid: fixtures.directory.cid, + recursive: false + })) await Promise.all([ expectPinned(ipfs, fixtures.directory.cid, pinTypes.direct), expectPinned(ipfs, fixtures.directory.files[0].cid, false) @@ -69,7 +149,10 @@ module.exports = (common, options) => { }) it('should recursively pin parent of direct pin', async () => { - await drain(ipfs.pin.add(fixtures.directory.files[0].cid, { recursive: false })) + await drain(ipfs.pin.add({ + cid: fixtures.directory.files[0].cid, + recursive: false + })) await drain(ipfs.pin.add(fixtures.directory.cid)) await Promise.all([ // file is pinned both directly and indirectly o.O @@ -80,7 +163,10 @@ module.exports = (common, options) => { it('should fail to directly pin a recursive pin', async () => { await drain(ipfs.pin.add(fixtures.directory.cid)) - return expect(drain(ipfs.pin.add(fixtures.directory.cid, { recursive: false }))) + return expect(drain(ipfs.pin.add({ + cid: fixtures.directory.cid, + recursive: false + }))) .to.eventually.be.rejected() .with(/already pinned recursively/) }) diff --git a/packages/interface-ipfs-core/src/pin/ls.js b/packages/interface-ipfs-core/src/pin/ls.js index 9345eabe10..0d67e3de0a 100644 --- a/packages/interface-ipfs-core/src/pin/ls.js +++ b/packages/interface-ipfs-core/src/pin/ls.js @@ -5,6 +5,7 @@ const { fixtures } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const all = require('it-all') const drain = require('it-drain') +const last = require('it-last') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -25,13 +26,16 @@ module.exports = (common, options) => { // two files wrapped in directories, only root CID pinned recursively const dir = fixtures.directory.files.map((file) => ({ path: file.path, content: file.data })) await drain(ipfs.add(dir, { pin: false, cidVersion: 0 })) - await drain(ipfs.pin.add(fixtures.directory.cid, { recursive: true })) + await drain(ipfs.pin.add(fixtures.directory.cid)) // a file (CID pinned recursively) await drain(ipfs.add(fixtures.files[0].data, { pin: false, cidVersion: 0 })) - await drain(ipfs.pin.add(fixtures.files[0].cid, { recursive: true })) + await drain(ipfs.pin.add(fixtures.files[0].cid)) // a single CID (pinned directly) await drain(ipfs.add(fixtures.files[1].data, { pin: false, cidVersion: 0 })) - await drain(ipfs.pin.add(fixtures.files[1].cid, { recursive: false })) + await drain(ipfs.pin.add({ + cid: fixtures.files[1].cid, + recursive: false + })) }) after(() => common.clean()) @@ -114,10 +118,10 @@ module.exports = (common, options) => { it('should list pins for a specific hash', async () => { const pinset = await all(ipfs.pin.ls(fixtures.files[0].cid)) expect(pinset).to.have.lengthOf(1) - expect(pinset).to.deep.include({ + expect(pinset).to.have.deep.members([{ type: 'recursive', cid: fixtures.files[0].cid - }) + }]) }) it('should throw an error on missing direct pins for existing path', () => { @@ -173,5 +177,27 @@ module.exports = (common, options) => { // TODO: go-ipfs does not return error codes // .with.property('code').that.equals('ERR_INVALID_PIN_TYPE') }) + + it('should list pins with comments', async () => { + const { cid } = await last(ipfs.add(`data-${Math.random()}`, { + pin: false + })) + + const comments = 'hello world' + + await drain(ipfs.pin.add({ + cid: cid, + recursive: false, + comments + })) + + const pinset = await all(ipfs.pin.ls(cid)) + + expect(pinset).to.have.deep.members([{ + type: 'direct', + cid: cid, + comments + }]) + }) }) } diff --git a/packages/interface-ipfs-core/src/pin/rm.js b/packages/interface-ipfs-core/src/pin/rm.js index e266af8995..4c7e05ad07 100644 --- a/packages/interface-ipfs-core/src/pin/rm.js +++ b/packages/interface-ipfs-core/src/pin/rm.js @@ -81,5 +81,13 @@ module.exports = (common, options) => { .to.eventually.be.rejected() .with(/is not pinned/) }) + + it('should pipe the output of ls to rm', async () => { + await drain(ipfs.pin.add(fixtures.directory.cid)) + + await drain(ipfs.pin.rm(ipfs.pin.ls({ type: 'recursive' }))) + + await expect(all(ipfs.pin.ls())).to.eventually.have.lengthOf(0) + }) }) } diff --git a/packages/interface-ipfs-core/src/pin/utils.js b/packages/interface-ipfs-core/src/pin/utils.js index dcb09c633a..db31c5834b 100644 --- a/packages/interface-ipfs-core/src/pin/utils.js +++ b/packages/interface-ipfs-core/src/pin/utils.js @@ -55,9 +55,9 @@ const expectPinned = async (ipfs, cid, type = pinTypes.all, pinned = true) => { expect(result).to.eql(pinned) } -async function isPinnedWithType (ipfs, path, type) { +async function isPinnedWithType (ipfs, cid, type) { try { - for await (const _ of ipfs.pin.ls(path, { type })) { // eslint-disable-line no-unused-vars + for await (const _ of ipfs.pin.ls(cid, { type })) { // eslint-disable-line no-unused-vars return true } return false diff --git a/packages/ipfs-http-client/src/pin/add.js b/packages/ipfs-http-client/src/pin/add.js index b42cbb03fd..948370a99c 100644 --- a/packages/ipfs-http-client/src/pin/add.js +++ b/packages/ipfs-http-client/src/pin/add.js @@ -2,23 +2,27 @@ const CID = require('cids') const configure = require('../lib/configure') +const normaliseInput = require('ipfs-utils/src/pins/normalise-input') module.exports = configure(({ ky }) => { - return async function * (paths, options) { - paths = Array.isArray(paths) ? paths : [paths] + return async function * (source, options) { options = options || {} - const searchParams = new URLSearchParams(options.searchParams) - paths.forEach(path => searchParams.append('arg', `${path}`)) - if (options.recursive != null) searchParams.set('recursive', options.recursive) + for await (const { path, recursive, comments } of normaliseInput(source)) { + const searchParams = new URLSearchParams(options.searchParams) + searchParams.append('arg', `${path}`) - const res = await ky.post('pin/add', { - timeout: options.timeout, - signal: options.signal, - headers: options.headers, - searchParams - }).json() + if (recursive != null) searchParams.set('recursive', recursive) + if (comments != null) searchParams.set('comments', comments) - yield * (res.Pins || []).map(cid => ({ cid: new CID(cid) })) + const res = await ky.post('pin/add', { + timeout: options.timeout, + signal: options.signal, + headers: options.headers, + searchParams + }).json() + + yield * (res.Pins || []).map(cid => ({ cid: new CID(cid) })) + } } }) diff --git a/packages/ipfs-http-client/src/pin/rm.js b/packages/ipfs-http-client/src/pin/rm.js index 0a2d46a22b..680abb4b00 100644 --- a/packages/ipfs-http-client/src/pin/rm.js +++ b/packages/ipfs-http-client/src/pin/rm.js @@ -2,23 +2,26 @@ const CID = require('cids') const configure = require('../lib/configure') +const normaliseInput = require('ipfs-utils/src/pins/normalise-input') module.exports = configure(({ ky }) => { - return async function * (paths, options) { - paths = Array.isArray(paths) ? paths : [paths] + return async function * (source, options) { options = options || {} - const searchParams = new URLSearchParams(options.searchParams) - paths.forEach(path => searchParams.append('arg', `${path}`)) - if (options.recursive != null) searchParams.set('recursive', options.recursive) + for await (const { path, recursive } of normaliseInput(source)) { + const searchParams = new URLSearchParams(options.searchParams) + searchParams.append('arg', `${path}`) - const res = await ky.post('pin/rm', { - timeout: options.timeout, - signal: options.signal, - headers: options.headers, - searchParams - }).json() + if (recursive != null) searchParams.set('recursive', recursive) - yield * (res.Pins || []).map(cid => ({ cid: new CID(cid) })) + const res = await ky.post('pin/rm', { + timeout: options.timeout, + signal: options.signal, + headers: options.headers, + searchParams + }).json() + + yield * (res.Pins || []).map(cid => ({ cid: new CID(cid) })) + } } }) diff --git a/packages/ipfs-http-client/test/interface.spec.js b/packages/ipfs-http-client/test/interface.spec.js index 53492cac7c..618a3e0e3f 100644 --- a/packages/ipfs-http-client/test/interface.spec.js +++ b/packages/ipfs-http-client/test/interface.spec.js @@ -350,7 +350,14 @@ describe('interface-ipfs-core tests', () => { tests.object(commonFactory) - tests.pin(commonFactory) + tests.pin(commonFactory, { + skip: [ + { + name: 'should list pins with comments', + reason: 'not implemented in go-ipfs' + } + ] + }) tests.ping(commonFactory, { skip: [ diff --git a/packages/ipfs-utils/package.json b/packages/ipfs-utils/package.json index fd2eff25e6..5eee3f44b0 100644 --- a/packages/ipfs-utils/package.json +++ b/packages/ipfs-utils/package.json @@ -29,6 +29,7 @@ "license": "MIT", "dependencies": { "buffer": "^5.4.2", + "cids": "^0.7.3", "err-code": "^2.0.0", "fs-extra": "^8.1.0", "is-electron": "^2.2.0", diff --git a/packages/ipfs-utils/src/pins/normalise-input.js b/packages/ipfs-utils/src/pins/normalise-input.js new file mode 100644 index 0000000000..4017a1913d --- /dev/null +++ b/packages/ipfs-utils/src/pins/normalise-input.js @@ -0,0 +1,124 @@ +'use strict' + +const errCode = require('err-code') +const CID = require('cids') + +/* + * Transform one of: + * + * ``` + * CID + * String + * { cid: CID recursive, comments } + * { path: String recursive, comments } + * Iterable + * Iterable + * Iterable<{ cid: CID recursive, comments }> + * Iterable<{ path: String recursive, comments }> + * AsyncIterable + * AsyncIterable + * AsyncIterable<{ cid: CID recursive, comments }> + * AsyncIterable<{ path: String recursive, comments }> + * ``` + * Into: + * + * ``` + * AsyncIterable<{ path: CID|String, recursive, comments }> + * ``` + * + * @param input Object + * @return AsyncIterable<{ path: CID|String, recursive, comments }> + */ +module.exports = function normaliseInput (input) { + // must give us something + if (input === null || input === undefined) { + throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT')) + } + + // CID|String + if (CID.isCID(input) || input instanceof String || typeof input === 'string') { + return (async function * () { // eslint-disable-line require-await + yield toPin({ cid: input }) + })() + } + + // { cid: CID recursive, comments } + if (input.cid != null || input.path != null) { + return (async function * () { // eslint-disable-line require-await + yield toPin(input) + })() + } + + // Iterable + if (input[Symbol.iterator]) { + return (async function * () { // eslint-disable-line require-await + const iterator = input[Symbol.iterator]() + const first = iterator.next() + if (first.done) return iterator + + // Iterable + if (CID.isCID(first.value) || first.value instanceof String || typeof first.value === 'string') { + yield toPin({ cid: first.value }) + for (const cid of iterator) { + yield toPin({ cid }) + } + return + } + + // Iterable<{ cid: CID recursive, comments }> + if (first.value.cid != null || first.value.path != null) { + yield toPin(first.value) + for (const obj of iterator) { + yield toPin(obj) + } + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + })() + } + + // AsyncIterable + if (input[Symbol.asyncIterator]) { + return (async function * () { + const iterator = input[Symbol.asyncIterator]() + const first = await iterator.next() + if (first.done) return iterator + + // AsyncIterable + if (CID.isCID(first.value) || first.value instanceof String || typeof first.value === 'string') { + yield toPin({ cid: first.value }) + for await (const cid of iterator) { + yield toPin({ cid }) + } + return + } + + // AsyncIterable<{ cid: CID|String recursive, comments }> + if (first.value.cid != null || first.value.path != null) { + yield toPin(first.value) + for await (const obj of iterator) { + yield toPin(obj) + } + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + })() + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') +} + +function toPin (input) { + const pin = { + path: input.cid || input.path, + recursive: input.recursive !== false + } + + if (input.comments != null) { + pin.comments = input.comments + } + + return pin +} diff --git a/packages/ipfs-utils/test/pins/normalise-input.spec.js b/packages/ipfs-utils/test/pins/normalise-input.spec.js new file mode 100644 index 0000000000..5f9ae32f76 --- /dev/null +++ b/packages/ipfs-utils/test/pins/normalise-input.spec.js @@ -0,0 +1,70 @@ +'use strict' + +/* eslint-env mocha */ +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const normalise = require('../../src/pins/normalise-input') +const all = require('it-all') +const CID = require('cids') + +chai.use(dirtyChai) +const expect = chai.expect + +const STRING = () => '/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn/path/to/file.txt' +const PLAIN_CID = () => new CID('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') +const OBJECT_CID = () => ({ cid: new CID('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'), recursive: true, comments: 'hello world' }) +const OBJECT_PATH = () => ({ path: '/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn/path/to/file.txt', recursive: true, comments: 'hello world' }) + +async function verifyNormalisation (input, withOptions) { + const result = await all(normalise(input)) + + expect(result).to.have.lengthOf(1) + expect(result[0]).to.have.property('path') + + if (withOptions) { + expect(result[0]).to.have.property('recursive', true) + expect(result[0]).to.have.property('comments', 'hello world') + } +} + +function iterableOf (thing) { + return [thing] +} + +function asyncIterableOf (thing) { + return (async function * () { // eslint-disable-line require-await + yield thing + }()) +} + +describe('pin normalise-input', function () { + function testInputType (content, name, withOptions) { + it(name, async function () { + await verifyNormalisation(content(), withOptions) + }) + + it(`Iterable<${name}>`, async function () { + await verifyNormalisation(iterableOf(content()), withOptions) + }) + + it(`AsyncIterable<${name}>`, async function () { + await verifyNormalisation(asyncIterableOf(content()), withOptions) + }) + } + + describe('String', () => { + testInputType(STRING, 'String') + }) + + describe('CID', () => { + testInputType(PLAIN_CID, 'CID') + }) + + describe('Object with CID', () => { + testInputType(OBJECT_CID, 'Object with CID', true) + }) + + describe('Object with path', () => { + testInputType(OBJECT_PATH, 'Object with path', true) + }) +}) diff --git a/packages/ipfs/src/cli/commands/pin/add.js b/packages/ipfs/src/cli/commands/pin/add.js index a1273e9c33..95b260c1ca 100644 --- a/packages/ipfs/src/cli/commands/pin/add.js +++ b/packages/ipfs/src/cli/commands/pin/add.js @@ -6,7 +6,7 @@ const { cidToString } = require('../../../utils/cid') module.exports = { command: 'add ', - describe: 'Pins object to local storage.', + describe: 'Pins object to local storage, preventing it from being garbage collected', builder: { recursive: { @@ -19,14 +19,19 @@ module.exports = { describe: 'Number base to display CIDs in.', type: 'string', choices: multibase.names + }, + comments: { + describe: 'A comment to add to the pin', + type: 'string', + alias: 'c' } }, - async handler ({ ctx, ipfsPath, recursive, cidBase }) { + async handler ({ ctx, ipfsPath, recursive, cidBase, comments }) { const { ipfs, print } = ctx const type = recursive ? 'recursive' : 'direct' - for await (const res of ipfs.pin.add(ipfsPath, { recursive })) { + for await (const res of ipfs.pin.add(ipfsPath.map(path => ({ path, recursive, comments })))) { print(`pinned ${cidToString(res.cid, { base: cidBase })} ${type}ly`) } } diff --git a/packages/ipfs/src/cli/commands/pin/ls.js b/packages/ipfs/src/cli/commands/pin/ls.js index a1d3ad7e2a..959ad93a2e 100644 --- a/packages/ipfs/src/cli/commands/pin/ls.js +++ b/packages/ipfs/src/cli/commands/pin/ls.js @@ -45,6 +45,10 @@ module.exports = { let line = cidToString(res.cid, { base: cidBase }) if (!quiet) { line += ` ${res.type}` + + if (res.comments) { + line += ` ${res.comments}` + } } print(line) } diff --git a/packages/ipfs/src/cli/commands/pin/rm.js b/packages/ipfs/src/cli/commands/pin/rm.js index 7ad2a4b640..59a946223e 100644 --- a/packages/ipfs/src/cli/commands/pin/rm.js +++ b/packages/ipfs/src/cli/commands/pin/rm.js @@ -6,7 +6,7 @@ const { cidToString } = require('../../../utils/cid') module.exports = { command: 'rm ', - describe: 'Removes the pinned object from local storage.', + describe: 'Unpins the corresponding block making it available for garbage collection', builder: { recursive: { @@ -25,7 +25,7 @@ module.exports = { async handler ({ ctx, ipfsPath, recursive, cidBase }) { const { ipfs, print } = ctx - for await (const res of ipfs.pin.rm(ipfsPath, { recursive })) { + for await (const res of ipfs.pin.rm(ipfsPath.map(path => ({ path, recursive })))) { print(`unpinned ${cidToString(res.cid, { base: cidBase })}`) } } diff --git a/packages/ipfs/src/core/components/pin/add.js b/packages/ipfs/src/core/components/pin/add.js index 8376e57b5f..328d4b4a3d 100644 --- a/packages/ipfs/src/core/components/pin/add.js +++ b/packages/ipfs/src/core/components/pin/add.js @@ -1,19 +1,21 @@ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { resolvePath, withTimeoutOption } = require('../../utils') +const { withTimeoutOption } = require('../../utils') const PinManager = require('./pin-manager') const { PinTypes } = PinManager +const normaliseInput = require('ipfs-utils/src/pins/normalise-input') +const { resolvePath } = require('../../utils') module.exports = ({ pinManager, gcLock, dag }) => { - return withTimeoutOption(async function * add (paths, options) { + return withTimeoutOption(async function * add (source, options) { options = options || {} - const recursive = options.recursive !== false - const cids = await resolvePath(dag, paths, { signal: options.signal }) const pinAdd = async function * () { - // verify that each hash can be pinned - for (const cid of cids) { + for await (const { path, recursive, comments } of normaliseInput(source)) { + const cid = await resolvePath(dag, path) + + // verify that each hash can be pinned const { reason } = await pinManager.isPinnedWithType(cid, [PinTypes.recursive, PinTypes.direct]) if (reason === 'recursive' && !recursive) { @@ -22,9 +24,9 @@ module.exports = ({ pinManager, gcLock, dag }) => { } if (recursive) { - await pinManager.pinRecursively(cid) + await pinManager.pinRecursively(cid, { comments }) } else { - await pinManager.pinDirectly(cid) + await pinManager.pinDirectly(cid, { comments }) } yield { cid } diff --git a/packages/ipfs/src/core/components/pin/ls.js b/packages/ipfs/src/core/components/pin/ls.js index 86e07c318c..9704c50977 100644 --- a/packages/ipfs/src/core/components/pin/ls.js +++ b/packages/ipfs/src/core/components/pin/ls.js @@ -1,9 +1,10 @@ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { resolvePath } = require('../../utils') const PinManager = require('./pin-manager') const { PinTypes } = PinManager +const normaliseInput = require('ipfs-utils/src/pins/normalise-input') +const { resolvePath } = require('../../utils') function toPin (type, cid, comments) { const output = { @@ -19,14 +20,14 @@ function toPin (type, cid, comments) { } module.exports = ({ pinManager, dag }) => { - return async function * ls (paths, options) { + return async function * ls (source, options) { options = options || {} let type = PinTypes.all - if (paths && paths.type) { - options = paths - paths = null + if (source && source.type) { + options = source + source = null } if (options.type) { @@ -40,35 +41,27 @@ module.exports = ({ pinManager, dag }) => { options.type = PinTypes.all } - if (paths) { - paths = Array.isArray(paths) ? paths : [paths] - + if (source) { // check the pinned state of specific hashes - const cids = await resolvePath(dag, paths, { signal: options.signal }) let noMatch = true - for (const cid of cids) { - const { reason, pinned, parent } = await pinManager.isPinnedWithType(cid, type) + for await (const { path } of normaliseInput(source)) { + const cid = await resolvePath(dag, path) + const { reason, pinned, parent, comments } = await pinManager.isPinnedWithType(cid, type) if (!pinned) { - throw new Error(`path '${paths}' is not pinned`) + throw new Error(`path '${path}' is not pinned`) } switch (reason) { case PinTypes.direct: case PinTypes.recursive: noMatch = false - yield { - type: reason, - cid - } + yield toPin(reason, cid, comments) break default: noMatch = false - yield { - type: `${PinTypes.indirect} through ${parent}`, - cid - } + yield toPin(`${PinTypes.indirect} through ${parent}`, cid, comments) } } @@ -87,10 +80,7 @@ module.exports = ({ pinManager, dag }) => { if (type === PinTypes.indirect || type === PinTypes.all) { for await (const cid of pinManager.indirectKeys(options)) { - yield { - type: PinTypes.indirect, - cid - } + yield toPin(PinTypes.indirect, cid) } } diff --git a/packages/ipfs/src/core/components/pin/pin-manager.js b/packages/ipfs/src/core/components/pin/pin-manager.js index 5669d4c6f7..f7419fc251 100644 --- a/packages/ipfs/src/core/components/pin/pin-manager.js +++ b/packages/ipfs/src/core/components/pin/pin-manager.js @@ -164,7 +164,8 @@ class PinManager { return { cid, pinned: true, - reason: pin.type + reason: pin.type, + comments: pin.comments } } } diff --git a/packages/ipfs/src/core/components/pin/rm.js b/packages/ipfs/src/core/components/pin/rm.js index 3a159ad667..92c44c9de2 100644 --- a/packages/ipfs/src/core/components/pin/rm.js +++ b/packages/ipfs/src/core/components/pin/rm.js @@ -1,20 +1,19 @@ 'use strict' -const { resolvePath } = require('../../utils') +const normaliseInput = require('ipfs-utils/src/pins/normalise-input') const { PinTypes } = require('./pin-manager') +const { resolvePath } = require('../../utils') module.exports = ({ pinManager, gcLock, dag }) => { - return async function * rm (paths, options) { + return async function * rm (source, options) { options = options || {} - const recursive = options.recursive == null ? true : options.recursive - const cids = await resolvePath(dag, paths, { signal: options.signal }) - const release = await gcLock.readLock() try { // verify that each hash can be unpinned - for (const cid of cids) { + for await (const { path, recursive } of normaliseInput(source)) { + const cid = await resolvePath(dag, path) const { pinned, reason } = await pinManager.isPinnedWithType(cid, PinTypes.all) if (!pinned) { diff --git a/packages/ipfs/src/core/utils.js b/packages/ipfs/src/core/utils.js index aa1ff7a9dd..bf84973710 100644 --- a/packages/ipfs/src/core/utils.js +++ b/packages/ipfs/src/core/utils.js @@ -23,6 +23,10 @@ exports.OFFLINE_ERROR = 'This command must be run in online mode. Try running \' * @throws on an invalid @param ipfsPath */ function parseIpfsPath (ipfsPath) { + if (!ipfsPath) { + throw new Error('invalid ipfs ref path') + } + ipfsPath = ipfsPath.replace(/^\/ipfs\//, '') const matched = ipfsPath.match(/([^/]+(?:\/[^/]+)*)\/?$/) if (!matched) { @@ -83,55 +87,44 @@ const normalizeCidPath = (path) => { * - /link/to/venus * - /ipfs//link/to/pluto * - multihash Buffer - * - Arrays of the above * * @param {Dag} dag The IPFS dag api - * @param {Array} ipfsPaths A single or collection of ipfs-paths + * @param {CID|String} ipfsPath A CID or IPFS path * @param {Object} [options] Optional options passed directly to dag.resolve - * @return {Promise>} + * @return {CID} */ -const resolvePath = async function (dag, ipfsPaths, options) { +const resolvePath = async function (dag, ipfsPath, options) { options = options || {} - if (!Array.isArray(ipfsPaths)) { - ipfsPaths = [ipfsPaths] + if (isIpfs.cid(ipfsPath)) { + return new CID(ipfsPath) } - const cids = [] - - for (const path of ipfsPaths) { - if (isIpfs.cid(path)) { - cids.push(new CID(path)) - continue - } + const { hash, links } = parseIpfsPath(ipfsPath) - const { hash, links } = parseIpfsPath(path) + if (!links.length) { + return new CID(hash) + } - if (!links.length) { - cids.push(new CID(hash)) - continue - } + let cid = new CID(hash) - let cid = new CID(hash) - try { - for await (const { value } of dag.resolve(path, options)) { - if (CID.isCID(value)) { - cid = value - } + try { + for await (const { value } of dag.resolve(ipfsPath, options)) { + if (CID.isCID(value)) { + cid = value } - } catch (err) { - // TODO: add error codes to IPLD - if (err.message.startsWith('Object has no property')) { - const linkName = err.message.replace('Object has no property \'', '').slice(0, -1) - err.message = `no link named "${linkName}" under ${cid}` - err.code = 'ERR_NO_LINK' - } - throw err } - cids.push(cid) + } catch (err) { + // TODO: add error codes to IPLD + if (err.message.startsWith('Object has no property')) { + const linkName = err.message.replace('Object has no property \'', '').slice(0, -1) + err.message = `no link named "${linkName}" under ${cid}` + err.code = 'ERR_NO_LINK' + } + throw err } - return cids + return cid } const mapFile = (file, options) => { diff --git a/packages/ipfs/src/http/api/resources/pin.js b/packages/ipfs/src/http/api/resources/pin.js index 65c32e213e..e59d3e0180 100644 --- a/packages/ipfs/src/http/api/resources/pin.js +++ b/packages/ipfs/src/http/api/resources/pin.js @@ -3,7 +3,6 @@ const multibase = require('multibase') const Joi = require('@hapi/joi') const Boom = require('@hapi/boom') -const isIpfs = require('is-ipfs') const { map, reduce } = require('streaming-iterables') const pipe = require('it-pipe') const ndjson = require('iterable-ndjson') @@ -28,7 +27,7 @@ function toPin (type, cid, comments) { } function parseArgs (request, h) { - let { arg } = request.query + let { arg, comments, recursive } = request.query if (!arg) { throw Boom.badRequest("Argument 'arg' is required") @@ -36,14 +35,13 @@ function parseArgs (request, h) { arg = Array.isArray(arg) ? arg : [arg] - arg.forEach(path => { - if (!isIpfs.ipfsPath(path) && !isIpfs.cid(path)) { - throw Boom.badRequest('invalid ipfs ref path') + return arg.map(arg => { + return { + path: arg, + comments, + recursive: Boolean(recursive !== 'false') } }) - - const recursive = request.query.recursive !== 'false' - return { path: arg, recursive } } exports.ls = { @@ -54,30 +52,27 @@ exports.ls = { }).unknown() }, - parseArgs (request, h) { - let { arg } = request.query + parseArgs: (request) => { + let { arg, type } = request.query if (arg) { arg = Array.isArray(arg) ? arg : [arg] - - arg.forEach(path => { - if (!isIpfs.ipfsPath(path) && !isIpfs.cid(path)) { - throw Boom.badRequest('invalid ipfs ref path') - } - }) + arg = arg.map(arg => arg) } - const type = request.query.type || 'all' - return { path: request.query.arg, type } + return { + source: arg, + type: type || 'all' + } }, async handler (request, h) { const { ipfs } = request.server.app - const { path, type } = request.pre.args + const { source, type } = request.pre.args if (!request.query.stream) { const res = await pipe( - ipfs.pin.ls(path, { type }), + ipfs.pin.ls(source, { type }), reduce((res, { type, cid, comments }) => { res.Keys[cidToString(cid, { base: request.query['cid-base'] })] = toPin(type, null, comments) return res @@ -88,7 +83,7 @@ exports.ls = { } return streamResponse(request, h, () => pipe( - ipfs.pin.ls(path, { type }), + ipfs.pin.ls(source, { type }), map(({ type, cid, comments }) => toPin(type, cidToString(cid, { base: request.query['cid-base'] }), comments)), ndjson.stringify )) @@ -106,11 +101,11 @@ exports.add = { async handler (request, h) { const { ipfs } = request.server.app - const { path, recursive } = request.pre.args + const source = request.pre.args let result try { - result = await all(ipfs.pin.add(path, { recursive })) + result = await all(ipfs.pin.add(source)) } catch (err) { if (err.message.includes('already pinned recursively')) { throw Boom.boomify(err, { statusCode: 400 }) @@ -135,11 +130,11 @@ exports.rm = { async handler (request, h) { const { ipfs } = request.server.app - const { path, recursive } = request.pre.args + const source = request.pre.args let result try { - result = await all(ipfs.pin.rm(path, { recursive })) + result = await all(ipfs.pin.rm(source)) } catch (err) { throw Boom.boomify(err, { message: 'Failed to remove pin' }) } diff --git a/packages/ipfs/test/cli/pin.js b/packages/ipfs/test/cli/pin.js index b667221799..656fee997a 100644 --- a/packages/ipfs/test/cli/pin.js +++ b/packages/ipfs/test/cli/pin.js @@ -33,7 +33,7 @@ describe('pin', () => { describe('rm', function () { it('recursively (default)', async () => { - ipfs.pin.rm.withArgs([pins.root], { recursive: true }).returns([{ + ipfs.pin.rm.withArgs([{ path: pins.root, recursive: true }]).returns([{ cid: new CID(pins.root) }]) @@ -42,7 +42,7 @@ describe('pin', () => { }) it('non recursively', async () => { - ipfs.pin.rm.withArgs([pins.root], { recursive: false }).returns([{ + ipfs.pin.rm.withArgs([{ path: pins.root, recursive: false }]).returns([{ cid: new CID(pins.root) }]) @@ -51,7 +51,7 @@ describe('pin', () => { }) it('non recursively (short option)', async () => { - ipfs.pin.rm.withArgs([pins.root], { recursive: false }).returns([{ + ipfs.pin.rm.withArgs([{ path: pins.root, recursive: false }]).returns([{ cid: new CID(pins.root) }]) @@ -60,7 +60,7 @@ describe('pin', () => { }) it('should rm and print CIDs encoded in specified base', async () => { - ipfs.pin.rm.withArgs([pins.root], { recursive: true }).returns([{ + ipfs.pin.rm.withArgs([{ path: pins.root, recursive: true }]).returns([{ cid: new CID(pins.root) }]) @@ -72,7 +72,7 @@ describe('pin', () => { describe('add', function () { it('recursively (default)', async () => { - ipfs.pin.add.withArgs([pins.root], { recursive: true }).returns([{ + ipfs.pin.add.withArgs([{ path: pins.root, recursive: true, comments: undefined }]).returns([{ cid: new CID(pins.root) }]) @@ -81,7 +81,7 @@ describe('pin', () => { }) it('non recursively', async () => { - ipfs.pin.add.withArgs([pins.root], { recursive: false }).returns([{ + ipfs.pin.add.withArgs([{ path: pins.root, recursive: false, comments: undefined }]).returns([{ cid: new CID(pins.root) }]) @@ -90,7 +90,7 @@ describe('pin', () => { }) it('non recursively (short option)', async () => { - ipfs.pin.add.withArgs([pins.root], { recursive: false }).returns([{ + ipfs.pin.add.withArgs([{ path: pins.root, recursive: false, comments: undefined }]).returns([{ cid: new CID(pins.root) }]) @@ -98,8 +98,26 @@ describe('pin', () => { expect(out).to.equal(`pinned ${pins.root} directly\n`) }) + it('with a comment', async () => { + ipfs.pin.add.withArgs([{ path: pins.root, recursive: true, comments: 'hello world' }]).returns([{ + cid: new CID(pins.root) + }]) + + const out = await cli(`pin add --comments 'hello world' ${pins.root}`, { ipfs }) + expect(out).to.equal(`pinned ${pins.root} recursively\n`) + }) + + it('with a comment (short option)', async () => { + ipfs.pin.add.withArgs([{ path: pins.root, recursive: true, comments: 'hello world' }]).returns([{ + cid: new CID(pins.root) + }]) + + const out = await cli(`pin add -c 'hello world' ${pins.root}`, { ipfs }) + expect(out).to.equal(`pinned ${pins.root} recursively\n`) + }) + it('should rm and print CIDs encoded in specified base', async () => { - ipfs.pin.add.withArgs([pins.root], { recursive: true }).returns([{ + ipfs.pin.add.withArgs([{ path: pins.root, recursive: true, comments: undefined }]).returns([{ cid: new CID(pins.root) }]) From 1d304db4392aff21c3dba8be6d374b8680ee7430 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 6 Mar 2020 16:42:00 +0000 Subject: [PATCH 07/40] fix: handle invalid paths in http routes --- packages/ipfs/src/core/utils.js | 9 +++--- packages/ipfs/src/http/api/resources/pin.js | 35 +++++++++++++++------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/packages/ipfs/src/core/utils.js b/packages/ipfs/src/core/utils.js index bf84973710..76d6df6cef 100644 --- a/packages/ipfs/src/core/utils.js +++ b/packages/ipfs/src/core/utils.js @@ -6,6 +6,7 @@ const TimeoutController = require('timeout-abort-controller') const anySignal = require('any-signal') const parseDuration = require('parse-duration') const { TimeoutError } = require('./errors') +const errCode = require('err-code') const ERR_BAD_PATH = 'ERR_BAD_PATH' exports.OFFLINE_ERROR = 'This command must be run in online mode. Try running \'ipfs daemon\' first.' @@ -24,13 +25,13 @@ exports.OFFLINE_ERROR = 'This command must be run in online mode. Try running \' */ function parseIpfsPath (ipfsPath) { if (!ipfsPath) { - throw new Error('invalid ipfs ref path') + throw errCode(new Error('invalid ipfs ref path'), ERR_BAD_PATH) } ipfsPath = ipfsPath.replace(/^\/ipfs\//, '') const matched = ipfsPath.match(/([^/]+(?:\/[^/]+)*)\/?$/) if (!matched) { - throw new Error('invalid ipfs ref path') + throw errCode(new Error('invalid ipfs ref path'), ERR_BAD_PATH) } const [hash, ...links] = matched[1].split('/') @@ -39,7 +40,7 @@ function parseIpfsPath (ipfsPath) { if (isIpfs.cid(hash)) { return { hash, links } } else { - throw new Error('invalid ipfs ref path') + throw errCode(new Error('invalid ipfs ref path'), ERR_BAD_PATH) } } @@ -57,7 +58,7 @@ const normalizePath = (pathStr) => { } else if (isIpfs.path(pathStr)) { return pathStr } else { - throw Object.assign(new Error(`invalid path: ${pathStr}`), { code: ERR_BAD_PATH }) + throw errCode(new Error(`invalid path: ${pathStr}`), ERR_BAD_PATH) } } diff --git a/packages/ipfs/src/http/api/resources/pin.js b/packages/ipfs/src/http/api/resources/pin.js index e59d3e0180..f7207a1a4d 100644 --- a/packages/ipfs/src/http/api/resources/pin.js +++ b/packages/ipfs/src/http/api/resources/pin.js @@ -71,15 +71,23 @@ exports.ls = { const { source, type } = request.pre.args if (!request.query.stream) { - const res = await pipe( - ipfs.pin.ls(source, { type }), - reduce((res, { type, cid, comments }) => { - res.Keys[cidToString(cid, { base: request.query['cid-base'] })] = toPin(type, null, comments) - return res - }, { Keys: {} }) - ) - - return h.response(res) + try { + const res = await pipe( + ipfs.pin.ls(source, { type }), + reduce((res, { type, cid, comments }) => { + res.Keys[cidToString(cid, { base: request.query['cid-base'] })] = toPin(type, null, comments) + return res + }, { Keys: {} }) + ) + + return h.response(res) + } catch (err) { + if (err.code === 'ERR_BAD_PATH') { + throw Boom.boomify(err, { statusCode: 400 }) + } + + throw Boom.boomify(err, { message: 'Failed to list pins' }) + } } return streamResponse(request, h, () => pipe( @@ -107,9 +115,14 @@ exports.add = { try { result = await all(ipfs.pin.add(source)) } catch (err) { + if (err.code === 'ERR_BAD_PATH') { + throw Boom.boomify(err, { statusCode: 400 }) + } + if (err.message.includes('already pinned recursively')) { throw Boom.boomify(err, { statusCode: 400 }) } + throw Boom.boomify(err, { message: 'Failed to add pin' }) } @@ -136,6 +149,10 @@ exports.rm = { try { result = await all(ipfs.pin.rm(source)) } catch (err) { + if (err.code === 'ERR_BAD_PATH') { + throw Boom.boomify(err, { statusCode: 400 }) + } + throw Boom.boomify(err, { message: 'Failed to remove pin' }) } From 8aeb8196ec12cf23bc5e6b64398e9bdad0944312 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 6 Mar 2020 17:11:22 +0000 Subject: [PATCH 08/40] chore: fix up tests --- packages/ipfs/test/core/utils.js | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/packages/ipfs/test/core/utils.js b/packages/ipfs/test/core/utils.js index 4c119f2115..18663dee18 100644 --- a/packages/ipfs/test/core/utils.js +++ b/packages/ipfs/test/core/utils.js @@ -128,35 +128,22 @@ describe('utils', () => { after(() => repo.teardown()) it('handles base58 hash format', async () => { - const hashes = await utils.resolvePath(node.dag, rootHash) + const hash = await utils.resolvePath(node.dag, rootHash) - expect(hashes.length).to.equal(1) - expect(hashes[0].buffer).to.deep.equal(rootMultihash) + expect(hash).to.have.property('buffer').that.deep.equals(rootMultihash) }) it('handles multihash format', async () => { - const hashes = await utils.resolvePath(node.dag, aboutMultihash) + const hash = await utils.resolvePath(node.dag, aboutMultihash) - expect(hashes.length).to.equal(1) - expect(hashes[0].buffer).to.deep.equal(aboutMultihash) + expect(hash).to.have.property('buffer').that.deep.equals(aboutMultihash) }) it('handles ipfs paths format', async function () { this.timeout(200 * 1000) - const hashes = await utils.resolvePath(node.dag, aboutPath) + const hash = await utils.resolvePath(node.dag, aboutPath) - expect(hashes.length).to.equal(1) - expect(hashes[0].buffer).to.deep.equal(aboutMultihash) - }) - - it('handles an array', async () => { - const paths = [rootHash, rootPath, rootMultihash] - const hashes = await utils.resolvePath(node.dag, paths) - - expect(hashes.length).to.equal(3) - expect(hashes[0].buffer).to.deep.equal(rootMultihash) - expect(hashes[1].buffer).to.deep.equal(rootMultihash) - expect(hashes[2].buffer).to.deep.equal(rootMultihash) + expect(hash).to.have.property('buffer').that.deep.equals(aboutMultihash) }) it('should error on invalid hashes', () => { From ab70c18b7087d8a39581089bd0abc56367e9950c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 10 Mar 2020 18:29:42 +0000 Subject: [PATCH 09/40] chore: responding to pr comments --- packages/interface-ipfs-core/SPEC/PIN.md | 4 ++++ packages/ipfs/src/core/components/pin/ls.js | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/interface-ipfs-core/SPEC/PIN.md b/packages/interface-ipfs-core/SPEC/PIN.md index 45279f21d5..4e6d6af17a 100644 --- a/packages/interface-ipfs-core/SPEC/PIN.md +++ b/packages/interface-ipfs-core/SPEC/PIN.md @@ -13,6 +13,10 @@ Where: - `source` is a [CID], an IPFS path or an object of the form `{ cid: CID, recursive: Boolean, comments: String }` or `{ path: String, recursive: Boolean, comments: String }` or an array or (async) iterable that yields any of the previous types + - Where this is an object, the fields mean the following: + - `cid`: The CID to pin + - `recursive`: Where the block that the CID resolves to contains links to other blocks, pin those too + - `comments`: A human readable string that can be used to add context to a pin - `options` is an object that can contain the following keys - `timeout` (`number`|`string`) - Throw an error if the request does not complete within the specified milliseconds timeout. If `timeout` is a string, the value is parsed as a [human readable duration](https://www.npmjs.com/package/parse-duration). There is no timeout by default. diff --git a/packages/ipfs/src/core/components/pin/ls.js b/packages/ipfs/src/core/components/pin/ls.js index 9704c50977..a1fa7db3e6 100644 --- a/packages/ipfs/src/core/components/pin/ls.js +++ b/packages/ipfs/src/core/components/pin/ls.js @@ -43,7 +43,7 @@ module.exports = ({ pinManager, dag }) => { if (source) { // check the pinned state of specific hashes - let noMatch = true + let matched = false for await (const { path } of normaliseInput(source)) { const cid = await resolvePath(dag, path) @@ -56,16 +56,16 @@ module.exports = ({ pinManager, dag }) => { switch (reason) { case PinTypes.direct: case PinTypes.recursive: - noMatch = false + matched = true yield toPin(reason, cid, comments) break default: - noMatch = false + matched = true yield toPin(`${PinTypes.indirect} through ${parent}`, cid, comments) } } - if (noMatch) { + if (!matched) { throw new Error('No match found') } From 1185cb99df0f9a1821a241e26343f950d555120a Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 10 Mar 2020 18:31:42 +0000 Subject: [PATCH 10/40] chore: responding to pr comments --- packages/ipfs/src/core/components/pin/pin-manager.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ipfs/src/core/components/pin/pin-manager.js b/packages/ipfs/src/core/components/pin/pin-manager.js index f7419fc251..a366d633e1 100644 --- a/packages/ipfs/src/core/components/pin/pin-manager.js +++ b/packages/ipfs/src/core/components/pin/pin-manager.js @@ -147,12 +147,12 @@ class PinManager { const result = await first(this.repo.pins.query({ prefix: toKey(cid), filters: [entry => { - const pin = cbor.decode(entry.value) - if (all) { return true } + const pin = cbor.decode(entry.value) + return types.includes(pin.type) }], limit: 1 From 591b7f241a73a80bf366db049fbbcfd96f183503 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 30 Jun 2020 18:04:09 +0100 Subject: [PATCH 11/40] chore: fix tests --- packages/interface-ipfs-core/src/pin/add.js | 6 +- packages/interface-ipfs-core/src/pin/ls.js | 20 +++- packages/interface-ipfs-core/src/pin/rm.js | 4 +- packages/interface-ipfs-core/src/pin/utils.js | 4 +- .../interface-ipfs-core/src/refs-local.js | 4 +- .../src/pins/normalise-input.js | 26 ++--- packages/ipfs-http-client/src/pin/add.js | 24 ++-- packages/ipfs-http-client/src/pin/ls.js | 13 ++- packages/ipfs-http-client/src/pin/rm.js | 13 +-- packages/ipfs/src/cli/commands/pin/add.js | 34 +++++- packages/ipfs/src/core/components/pin/add.js | 13 +-- packages/ipfs/src/core/components/pin/ls.js | 27 +++-- .../src/core/components/pin/pin-manager.js | 13 ++- packages/ipfs/src/core/components/pin/rm.js | 17 +-- packages/ipfs/src/http/api/resources/pin.js | 24 ++-- packages/ipfs/src/http/utils/joi.js | 13 +++ packages/ipfs/test/cli/pin.js | 106 ++++++++++++------ 17 files changed, 223 insertions(+), 138 deletions(-) diff --git a/packages/interface-ipfs-core/src/pin/add.js b/packages/interface-ipfs-core/src/pin/add.js index cf6c952ef3..177338c14b 100644 --- a/packages/interface-ipfs-core/src/pin/add.js +++ b/packages/interface-ipfs-core/src/pin/add.js @@ -52,7 +52,7 @@ module.exports = (common, options) => { async function testAddInput (source) { const pinset = await all(ipfs.pin.add(source)) - expect(pinset.map(p => p.cid)).to.have.deep.members([ + expect(pinset).to.have.deep.members([ fixtures.files[0].cid, fixtures.files[1].cid ]) @@ -60,7 +60,7 @@ module.exports = (common, options) => { it('should add a CID and return the added CID', async () => { const pinset = await all(ipfs.pin.add(fixtures.files[0].cid)) - expect(pinset.map(p => p.cid)).to.deep.include(fixtures.files[0].cid) + expect(pinset).to.deep.include(fixtures.files[0].cid) }) it('should add a pin with options and return the added CID', async () => { @@ -68,7 +68,7 @@ module.exports = (common, options) => { cid: fixtures.files[0].cid, recursive: false })) - expect(pinset.map(p => p.cid)).to.deep.include(fixtures.files[0].cid) + expect(pinset).to.deep.include(fixtures.files[0].cid) }) it('should add an array of CIDs', () => { diff --git a/packages/interface-ipfs-core/src/pin/ls.js b/packages/interface-ipfs-core/src/pin/ls.js index fe95cc25f5..17cf34e651 100644 --- a/packages/interface-ipfs-core/src/pin/ls.js +++ b/packages/interface-ipfs-core/src/pin/ls.js @@ -42,7 +42,7 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when listing pins', () => { - return testTimeout(() => ipfs.pin.ls({ + return testTimeout(() => ipfs.pin.ls(undefined, { timeout: 1 })) }) @@ -185,17 +185,27 @@ module.exports = (common, options) => { // .with.property('code').that.equals('ERR_INVALID_PIN_TYPE') }) - it('should list pins with comments', async () => { + it('should list pins with metadata', async () => { const { cid } = await last(ipfs.add(`data-${Math.random()}`, { pin: false })) - const comments = 'hello world' + const metadata = { + key: 'value', + one: 2, + array: [{ + thing: 'subthing' + }], + obj: { + foo: 'bar', + baz: ['qux'] + } + } await drain(ipfs.pin.add({ cid: cid, recursive: false, - comments + metadata })) const pinset = await all(ipfs.pin.ls(cid)) @@ -203,7 +213,7 @@ module.exports = (common, options) => { expect(pinset).to.have.deep.members([{ type: 'direct', cid: cid, - comments + metadata }]) }) }) diff --git a/packages/interface-ipfs-core/src/pin/rm.js b/packages/interface-ipfs-core/src/pin/rm.js index ab7ff03cdb..1ac2ee4294 100644 --- a/packages/interface-ipfs-core/src/pin/rm.js +++ b/packages/interface-ipfs-core/src/pin/rm.js @@ -56,7 +56,7 @@ module.exports = (common, options) => { await drain(ipfs.pin.add(fixtures.directory.cid)) const removedPinset = await all(ipfs.pin.rm(fixtures.directory.cid, { recursive: true })) - expect(removedPinset.map(p => p.cid)).to.deep.equal([fixtures.directory.cid]) + expect(removedPinset).to.deep.equal([fixtures.directory.cid]) const pinset = await all(ipfs.pin.ls({ type: 'recursive' })) expect(pinset).to.not.deep.include({ @@ -69,7 +69,7 @@ module.exports = (common, options) => { await drain(ipfs.pin.add(fixtures.directory.cid, { recursive: false })) const removedPinset = await all(ipfs.pin.rm(fixtures.directory.cid, { recursive: false })) - expect(removedPinset.map(p => p.cid)).to.deep.equal([fixtures.directory.cid]) + expect(removedPinset).to.deep.equal([fixtures.directory.cid]) const pinset = await all(ipfs.pin.ls({ type: 'direct' })) expect(pinset.map(p => p.cid)).to.not.deep.include(fixtures.directory.cid) diff --git a/packages/interface-ipfs-core/src/pin/utils.js b/packages/interface-ipfs-core/src/pin/utils.js index db31c5834b..a1ad57ac8e 100644 --- a/packages/interface-ipfs-core/src/pin/utils.js +++ b/packages/interface-ipfs-core/src/pin/utils.js @@ -36,11 +36,11 @@ const fixtures = Object.freeze({ }) const clearPins = async (ipfs) => { - for await (const { cid } of ipfs.pin.ls({ type: pinTypes.recursive })) { + for await (const { cid } of ipfs.pin.ls(null, { type: pinTypes.recursive })) { await drain(ipfs.pin.rm(cid)) } - for await (const { cid } of ipfs.pin.ls({ type: pinTypes.direct })) { + for await (const { cid } of ipfs.pin.ls(null, { type: pinTypes.direct })) { await drain(ipfs.pin.rm(cid)) } } diff --git a/packages/interface-ipfs-core/src/refs-local.js b/packages/interface-ipfs-core/src/refs-local.js index d20851dee8..af39c8163c 100644 --- a/packages/interface-ipfs-core/src/refs-local.js +++ b/packages/interface-ipfs-core/src/refs-local.js @@ -48,9 +48,7 @@ module.exports = (common, options) => { const imported = await all(importer(dirs, ipfs.block)) // otherwise go-ipfs doesn't show them in the local refs - await Promise.all( - imported.map(i => ipfs.pin.add(i.cid)) - ) + await drain(ipfs.pin.add(imported.map(i => i.cid))) const refs = await all(ipfs.refs.local()) const cids = refs.map(r => r.ref) diff --git a/packages/ipfs-core-utils/src/pins/normalise-input.js b/packages/ipfs-core-utils/src/pins/normalise-input.js index 4017a1913d..9cf591932a 100644 --- a/packages/ipfs-core-utils/src/pins/normalise-input.js +++ b/packages/ipfs-core-utils/src/pins/normalise-input.js @@ -9,25 +9,25 @@ const CID = require('cids') * ``` * CID * String - * { cid: CID recursive, comments } - * { path: String recursive, comments } + * { cid: CID recursive, metadata } + * { path: String recursive, metadata } * Iterable * Iterable - * Iterable<{ cid: CID recursive, comments }> - * Iterable<{ path: String recursive, comments }> + * Iterable<{ cid: CID recursive, metadata }> + * Iterable<{ path: String recursive, metadata }> * AsyncIterable * AsyncIterable - * AsyncIterable<{ cid: CID recursive, comments }> - * AsyncIterable<{ path: String recursive, comments }> + * AsyncIterable<{ cid: CID recursive, metadata }> + * AsyncIterable<{ path: String recursive, metadata }> * ``` * Into: * * ``` - * AsyncIterable<{ path: CID|String, recursive, comments }> + * AsyncIterable<{ path: CID|String, recursive, metadata }> * ``` * * @param input Object - * @return AsyncIterable<{ path: CID|String, recursive, comments }> + * @return AsyncIterable<{ path: CID|String, recursive, metadata }> */ module.exports = function normaliseInput (input) { // must give us something @@ -42,7 +42,7 @@ module.exports = function normaliseInput (input) { })() } - // { cid: CID recursive, comments } + // { cid: CID recursive, metadata } if (input.cid != null || input.path != null) { return (async function * () { // eslint-disable-line require-await yield toPin(input) @@ -65,7 +65,7 @@ module.exports = function normaliseInput (input) { return } - // Iterable<{ cid: CID recursive, comments }> + // Iterable<{ cid: CID recursive, metadata }> if (first.value.cid != null || first.value.path != null) { yield toPin(first.value) for (const obj of iterator) { @@ -94,7 +94,7 @@ module.exports = function normaliseInput (input) { return } - // AsyncIterable<{ cid: CID|String recursive, comments }> + // AsyncIterable<{ cid: CID|String recursive, metadata }> if (first.value.cid != null || first.value.path != null) { yield toPin(first.value) for await (const obj of iterator) { @@ -116,8 +116,8 @@ function toPin (input) { recursive: input.recursive !== false } - if (input.comments != null) { - pin.comments = input.comments + if (input.metadata != null) { + pin.metadata = input.metadata } return pin diff --git a/packages/ipfs-http-client/src/pin/add.js b/packages/ipfs-http-client/src/pin/add.js index 6ea95fe523..dc556a51ef 100644 --- a/packages/ipfs-http-client/src/pin/add.js +++ b/packages/ipfs-http-client/src/pin/add.js @@ -2,25 +2,35 @@ const CID = require('cids') const configure = require('../lib/configure') -const normaliseInput = require('ipfs-utils/src/pins/normalise-input') +const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') const toUrlSearchParams = require('../lib/to-url-search-params') module.exports = configure(api => { return async function * (source, options = {}) { - for await (const { path, recursive, comments } of normaliseInput(source)) { - const res = await (await api.post('pin/add', { + for await (const { path, recursive, metadata } of normaliseInput(source)) { + const res = await api.post('pin/add', { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ arg: path, recursive, - comments, - ...options + metadata: metadata ? JSON.stringify(metadata) : undefined, + ...options, + stream: true }), headers: options.headers - })).json() + }) - yield * (res.Pins || []).map(cid => ({ cid: new CID(cid) })) + for await (const pin of res.ndjson()) { + if (pin.Pins) { // non-streaming response + for (const cid of pin.Pins) { + yield new CID(cid) + } + continue + } + + yield new CID(pin) + } } } }) diff --git a/packages/ipfs-http-client/src/pin/ls.js b/packages/ipfs-http-client/src/pin/ls.js index 20d248dcd4..46e6c4684a 100644 --- a/packages/ipfs-http-client/src/pin/ls.js +++ b/packages/ipfs-http-client/src/pin/ls.js @@ -4,14 +4,14 @@ const CID = require('cids') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') -function toPin (type, cid, comments) { +function toPin (type, cid, metadata) { const pin = { type, cid: new CID(cid) } - if (comments) { - pin.comments = comments + if (metadata) { + pin.metadata = metadata } return pin @@ -37,14 +37,15 @@ module.exports = configure(api => { headers: options.headers }) - for await (const pin of ndjson(toIterable(res.body))) { + for await (const pin of res.ndjson()) { if (pin.Keys) { // non-streaming response for (const cid of Object.keys(pin.Keys)) { - yield toPin(pin.Keys[cid].Type, cid, pin.Keys[cid].Comments) + yield toPin(pin.Keys[cid].Type, cid, pin.Keys[cid].Metadata) } return } - yield toPin(pin.Type, pin.Cid, pin.Comments) + + yield toPin(pin.Type, pin.Cid, pin.Metadata) } } }) diff --git a/packages/ipfs-http-client/src/pin/rm.js b/packages/ipfs-http-client/src/pin/rm.js index 579b7449a8..e2d84babb2 100644 --- a/packages/ipfs-http-client/src/pin/rm.js +++ b/packages/ipfs-http-client/src/pin/rm.js @@ -2,7 +2,7 @@ const CID = require('cids') const configure = require('../lib/configure') -const normaliseInput = require('ipfs-utils/src/pins/normalise-input') +const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') const toUrlSearchParams = require('../lib/to-url-search-params') module.exports = configure(api => { @@ -15,7 +15,7 @@ module.exports = configure(api => { if (recursive != null) searchParams.set('recursive', recursive) - const res = await ky.post('pin/rm', { + const res = await api.post('pin/rm', { timeout: options.timeout, signal: options.signal, headers: options.headers, @@ -26,13 +26,12 @@ module.exports = configure(api => { }) }) - for await (const pin of ndjson(toIterable(res.body))) { + for await (const pin of res.ndjson()) { if (pin.Pins) { // non-streaming response - yield * res.Pins.map(cid => ({ cid: new CID(cid) })) - - return + yield * pin.Pins.map(cid => new CID(cid)) + continue } - yield { cid: new CID(pin.cid) } + yield new CID(pin) } } } diff --git a/packages/ipfs/src/cli/commands/pin/add.js b/packages/ipfs/src/cli/commands/pin/add.js index c6014f64e4..cbf1c53c71 100644 --- a/packages/ipfs/src/cli/commands/pin/add.js +++ b/packages/ipfs/src/cli/commands/pin/add.js @@ -3,6 +3,7 @@ const multibase = require('multibase') const { cidToString } = require('../../../utils/cid') const parseDuration = require('parse-duration') +const { JSON } = require('ipfs-utils/src/globalthis') module.exports = { command: 'add ', @@ -25,18 +26,41 @@ module.exports = { type: 'string', coerce: parseDuration }, - comments: { - describe: 'A comment to add to the pin', + metadata: { + describe: 'Metadata to add to the pin', type: 'string', - alias: 'c' + alias: 'm', + coerce: (val) => { + if (!val) { + return + } + + const output = {} + + val.split(',').forEach(line => { + const parts = line.split('=') + output[parts[0]] = parts[1] + }) + + return output + } + }, + 'metadata-json': { + describe: 'Metadata to add to the pin in JSON format', + type: 'string', + coerce: JSON.parse } }, - async handler ({ ctx, ipfsPath, recursive, cidBase, timeout, comments }) { + async handler ({ ctx, ipfsPath, recursive, cidBase, timeout, metadata, metadataJson }) { const { ipfs, print } = ctx const type = recursive ? 'recursive' : 'direct' - for await (const res of ipfs.pin.add(ipfsPath.map(path => ({ path, recursive, comments })), { timeout })) { + if (metadataJson) { + metadata = metadataJson + } + + for await (const res of ipfs.pin.add(ipfsPath.map(path => ({ path, recursive, metadata })), { timeout })) { print(`pinned ${cidToString(res.cid, { base: cidBase })} ${type}ly`) } } diff --git a/packages/ipfs/src/core/components/pin/add.js b/packages/ipfs/src/core/components/pin/add.js index 328d4b4a3d..2b4bd40b83 100644 --- a/packages/ipfs/src/core/components/pin/add.js +++ b/packages/ipfs/src/core/components/pin/add.js @@ -1,18 +1,17 @@ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { withTimeoutOption } = require('../../utils') +const { resolvePath, withTimeoutOption } = require('../../utils') const PinManager = require('./pin-manager') const { PinTypes } = PinManager -const normaliseInput = require('ipfs-utils/src/pins/normalise-input') -const { resolvePath } = require('../../utils') +const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') module.exports = ({ pinManager, gcLock, dag }) => { return withTimeoutOption(async function * add (source, options) { options = options || {} const pinAdd = async function * () { - for await (const { path, recursive, comments } of normaliseInput(source)) { + for await (const { path, recursive, metadata } of normaliseInput(source)) { const cid = await resolvePath(dag, path) // verify that each hash can be pinned @@ -24,12 +23,12 @@ module.exports = ({ pinManager, gcLock, dag }) => { } if (recursive) { - await pinManager.pinRecursively(cid, { comments }) + await pinManager.pinRecursively(cid, { metadata }) } else { - await pinManager.pinDirectly(cid, { comments }) + await pinManager.pinDirectly(cid, { metadata }) } - yield { cid } + yield cid } } diff --git a/packages/ipfs/src/core/components/pin/ls.js b/packages/ipfs/src/core/components/pin/ls.js index 441127e99f..c3b202f03e 100644 --- a/packages/ipfs/src/core/components/pin/ls.js +++ b/packages/ipfs/src/core/components/pin/ls.js @@ -3,18 +3,17 @@ const PinManager = require('./pin-manager') const { PinTypes } = PinManager -const normaliseInput = require('ipfs-utils/src/pins/normalise-input') -const { resolvePath } = require('../../utils') -const { withTimeoutOption } = require('../../utils') +const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const { resolvePath, withTimeoutOption } = require('../../utils') -function toPin (type, cid, comments) { +function toPin (type, cid, metadata) { const output = { type, cid } - if (comments) { - output.comments = comments + if (metadata) { + output.metadata = metadata } return output @@ -48,7 +47,7 @@ module.exports = ({ pinManager, dag }) => { for await (const { path } of normaliseInput(source)) { const cid = await resolvePath(dag, path) - const { reason, pinned, parent, comments } = await pinManager.isPinnedWithType(cid, type) + const { reason, pinned, parent, metadata } = await pinManager.isPinnedWithType(cid, type) if (!pinned) { throw new Error(`path '${path}' is not pinned`) @@ -58,11 +57,11 @@ module.exports = ({ pinManager, dag }) => { case PinTypes.direct: case PinTypes.recursive: matched = true - yield toPin(reason, cid, comments) + yield toPin(reason, cid, metadata) break default: matched = true - yield toPin(`${PinTypes.indirect} through ${parent}`, cid, comments) + yield toPin(`${PinTypes.indirect} through ${parent}`, cid, metadata) } } @@ -74,8 +73,8 @@ module.exports = ({ pinManager, dag }) => { } if (type === PinTypes.recursive || type === PinTypes.all) { - for await (const { cid, comments } of pinManager.recursiveKeys()) { - yield toPin(PinTypes.recursive, cid, comments) + for await (const { cid, metadata } of pinManager.recursiveKeys()) { + yield toPin(PinTypes.recursive, cid, metadata) } } @@ -86,9 +85,9 @@ module.exports = ({ pinManager, dag }) => { } if (type === PinTypes.direct || type === PinTypes.all) { - for await (const { cid, comments } of pinManager.directKeys()) { - yield toPin(PinTypes.direct, cid, comments) + for await (const { cid, metadata } of pinManager.directKeys()) { + yield toPin(PinTypes.direct, cid, metadata) } } - } + }) } diff --git a/packages/ipfs/src/core/components/pin/pin-manager.js b/packages/ipfs/src/core/components/pin/pin-manager.js index a366d633e1..2b0e894bb0 100644 --- a/packages/ipfs/src/core/components/pin/pin-manager.js +++ b/packages/ipfs/src/core/components/pin/pin-manager.js @@ -9,6 +9,7 @@ const debug = require('debug') const first = require('it-first') const all = require('it-all') const cbor = require('cbor') +const multibase = require('multibase') // arbitrary limit to the number of concurrent dag operations // const WALK_DAG_CONCURRENCY_LIMIT = 300 @@ -21,7 +22,7 @@ function invalidPinTypeErr (type) { } function toKey (cid) { - return '/' + cid.multihash.toString('base64') + return '/' + multibase.encode('base32', cid.multihash).toString().substring(1) } const PinTypes = { @@ -62,7 +63,7 @@ class PinManager { return this.repo.pins.put(toKey(cid), cbor.encode({ cid: cid.buffer, type: PinTypes.direct, - comments: options.comments + metadata: options.metadata })) } @@ -76,7 +77,7 @@ class PinManager { await this.repo.pins.put(toKey(cid), cbor.encode({ cid: cid.buffer, type: PinTypes.recursive, - comments: options.comments + metadata: options.metadata })) } @@ -92,7 +93,7 @@ class PinManager { yield { cid: new CID(pin.cid), - comments: pin.comments + metadata: pin.metadata } } } @@ -109,7 +110,7 @@ class PinManager { yield { cid: new CID(pin.cid), - comments: pin.comments + metadata: pin.metadata } } } @@ -165,7 +166,7 @@ class PinManager { cid, pinned: true, reason: pin.type, - comments: pin.comments + metadata: pin.metadata } } } diff --git a/packages/ipfs/src/core/components/pin/rm.js b/packages/ipfs/src/core/components/pin/rm.js index b496f03fc1..a406bac7f5 100644 --- a/packages/ipfs/src/core/components/pin/rm.js +++ b/packages/ipfs/src/core/components/pin/rm.js @@ -1,16 +1,11 @@ 'use strict' -const normaliseInput = require('ipfs-utils/src/pins/normalise-input') -const errCode = require('err-code') -const multibase = require('multibase') -const { parallelMap, collect } = require('streaming-iterables') -const pipe = require('it-pipe') +const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') const { resolvePath, withTimeoutOption } = require('../../utils') const { PinTypes } = require('./pin-manager') -const { resolvePath } = require('../../utils') module.exports = ({ pinManager, gcLock, dag }) => { - return withTimeoutOption(async function rm (source, options = {}) { + return withTimeoutOption(async function * rm (source, options = {}) { const release = await gcLock.readLock() try { @@ -31,17 +26,13 @@ module.exports = ({ pinManager, gcLock, dag }) => { await pinManager.unpin(cid) - yield { - cid - } + yield cid break case (PinTypes.direct): await pinManager.unpin(cid) - yield { - cid - } + yield cid break default: diff --git a/packages/ipfs/src/http/api/resources/pin.js b/packages/ipfs/src/http/api/resources/pin.js index ee71f5b2ee..098efd886f 100644 --- a/packages/ipfs/src/http/api/resources/pin.js +++ b/packages/ipfs/src/http/api/resources/pin.js @@ -9,7 +9,7 @@ const { cidToString } = require('../../../utils/cid') const streamResponse = require('../../utils/stream-response') const all = require('it-all') -function toPin (type, cid, comments) { +function toPin (type, cid, metadata) { const output = { Type: type } @@ -18,8 +18,8 @@ function toPin (type, cid, comments) { output.Cid = cid } - if (comments) { - output.Comments = comments + if (metadata) { + output.Metadata = metadata } return output @@ -76,8 +76,8 @@ exports.ls = { signal, timeout }), - reduce((res, { type, cid }) => { - res.Keys[cidToString(cid, { base: cidBase })] = { Type: type } + reduce((res, { type, cid, metadata }) => { + res.Keys[cidToString(cid, { base: cidBase })] = toPin(type, metadata) return res }, { Keys: {} }) ) @@ -91,7 +91,7 @@ exports.ls = { signal, timeout }), - map(({ type, cid }) => ({ Type: type, Cid: cidToString(cid, { base: cidBase }) })), + map(({ type, cid, metadata }) => toPin(type, cidToString(cid, { base: cidBase }), metadata)), ndjson.stringify )) } @@ -108,7 +108,8 @@ exports.add = { cids: Joi.array().single().items(Joi.cid()).min(1).required(), recursive: Joi.boolean().default(true), cidBase: Joi.cidBase(), - timeout: Joi.timeout() + timeout: Joi.timeout(), + metadata: Joi.json() }) .rename('cid-base', 'cidBase', { override: true, @@ -134,13 +135,14 @@ exports.add = { cids, recursive, cidBase, - timeout + timeout, + metadata } } = request let result try { - result = await all(ipfs.pin.add(cids.map(cid => ({ cid, recursive })), { + result = await all(ipfs.pin.add(cids.map(cid => ({ cid, recursive, metadata })), { signal, timeout })) @@ -157,7 +159,7 @@ exports.add = { } return h.response({ - Pins: result.map(obj => cidToString(obj.cid, { base: cidBase })) + Pins: result.map(cid => cidToString(cid, { base: cidBase })) }) } } @@ -218,7 +220,7 @@ exports.rm = { } return h.response({ - Pins: result.map(obj => cidToString(obj.cid, { base: cidBase })) + Pins: result.map(cid => cidToString(cid, { base: cidBase })) }) } } diff --git a/packages/ipfs/src/http/utils/joi.js b/packages/ipfs/src/http/utils/joi.js index 31775f5abc..be4cb733e5 100644 --- a/packages/ipfs/src/http/utils/joi.js +++ b/packages/ipfs/src/http/utils/joi.js @@ -144,4 +144,17 @@ module.exports = Joi return value } } + }, + (joi) => { + return { + name: 'json', + base: joi.string(), + pre (value, state, options) { + if (!value) { + return + } + + return JSON.parse(value) + } + } }) diff --git a/packages/ipfs/test/cli/pin.js b/packages/ipfs/test/cli/pin.js index 8af307a52d..ee6f258f89 100644 --- a/packages/ipfs/test/cli/pin.js +++ b/packages/ipfs/test/cli/pin.js @@ -32,16 +32,19 @@ describe('pin', () => { }) describe('rm', function () { + const defaultPinOptions = { + recursive: true + } + const defaultOptions = { - recursive: true, timeout: undefined } it('recursively (default)', async () => { ipfs.pin.rm.withArgs([{ - ...defaultOptions, + ...defaultPinOptions, path: pins.root - }]).returns([{ + }], defaultOptions).returns([{ cid: new CID(pins.root) }]) @@ -51,10 +54,10 @@ describe('pin', () => { it('non recursively', async () => { ipfs.pin.rm.withArgs([{ - ...defaultOptions, + ...defaultPinOptions, path: pins.root, recursive: false - }]).returns([{ + }], defaultOptions).returns([{ cid: new CID(pins.root) }]) @@ -64,10 +67,10 @@ describe('pin', () => { it('non recursively (short option)', async () => { ipfs.pin.rm.withArgs([{ - ...defaultOptions, + ...defaultPinOptions, path: pins.root, recursive: false - }]).returns([{ + }], defaultOptions).returns([{ cid: new CID(pins.root) }]) @@ -77,10 +80,9 @@ describe('pin', () => { it('should rm and print CIDs encoded in specified base', async () => { ipfs.pin.rm.withArgs([{ - ...defaultOptions, - path: pins.root, - recursive: true - }]).returns([{ + ...defaultPinOptions, + path: pins.root + }], defaultOptions).returns([{ cid: new CID(pins.root) }]) @@ -89,11 +91,14 @@ describe('pin', () => { expect(out).to.eql(`unpinned ${b64CidStr}\n`) }) - it('recursively with timeout', async () => { - ipfs.pin.rm.withArgs([pins.root], { + it('with timeout', async () => { + ipfs.pin.rm.withArgs([{ + ...defaultPinOptions, + path: pins.root + }], { ...defaultOptions, timeout: 1000 - }).resolves([{ + }).returns([{ cid: new CID(pins.root) }]) @@ -103,15 +108,20 @@ describe('pin', () => { }) describe('add', function () { - const defaultOptions = { + const defaultPinOptions = { recursive: true, + metadata: undefined + } + + const defaultOptions = { timeout: undefined } it('recursively (default)', async () => { ipfs.pin.add.withArgs([{ - ...defaultOptions, - path: pins.root, recursive: true, comments: undefined }]).returns([{ + ...defaultPinOptions, + path: pins.root + }], defaultOptions).returns([{ cid: new CID(pins.root) }]) @@ -121,11 +131,10 @@ describe('pin', () => { it('non recursively', async () => { ipfs.pin.add.withArgs([{ - ...defaultOptions, + ...defaultPinOptions, path: pins.root, - recursive: false, - comments: undefined - }]).returns([{ + recursive: false + }], defaultOptions).returns([{ cid: new CID(pins.root) }]) @@ -135,11 +144,10 @@ describe('pin', () => { it('non recursively (short option)', async () => { ipfs.pin.add.withArgs([{ - ...defaultOptions, + ...defaultPinOptions, path: pins.root, - recursive: false, - comments: undefined - }]).returns([{ + recursive: false + }], defaultOptions).returns([{ cid: new CID(pins.root) }]) @@ -147,21 +155,48 @@ describe('pin', () => { expect(out).to.equal(`pinned ${pins.root} directly\n`) }) - it('with a comment', async () => { - ipfs.pin.add.withArgs([{ path: pins.root, recursive: true, comments: 'hello world' }]).returns([{ + it('with metadata', async () => { + ipfs.pin.add.withArgs([{ + ...defaultPinOptions, + path: pins.root, + metadata: { + key: 'value' + } + }], defaultOptions).returns([{ + cid: new CID(pins.root) + }]) + + const out = await cli(`pin add --metadata key=value ${pins.root}`, { ipfs }) + expect(out).to.equal(`pinned ${pins.root} recursively\n`) + }) + + it('with a metadata (short option)', async () => { + ipfs.pin.add.withArgs([{ + ...defaultPinOptions, + path: pins.root, + metadata: { + key: 'value' + } + }], defaultOptions).returns([{ cid: new CID(pins.root) }]) - const out = await cli(`pin add --comments 'hello world' ${pins.root}`, { ipfs }) + const out = await cli(`pin add -m key=value ${pins.root}`, { ipfs }) expect(out).to.equal(`pinned ${pins.root} recursively\n`) }) - it('with a comment (short option)', async () => { - ipfs.pin.add.withArgs([{ path: pins.root, recursive: true, comments: 'hello world' }]).returns([{ + it('with json metadata', async () => { + ipfs.pin.add.withArgs([{ + ...defaultPinOptions, + path: pins.root, + metadata: { + key: 'value' + } + }], defaultOptions).returns([{ cid: new CID(pins.root) }]) - const out = await cli(`pin add -c 'hello world' ${pins.root}`, { ipfs }) + const out = await cli(`pin add --metadata-json '{"key":"value"}' ${pins.root}`, { ipfs }) expect(out).to.equal(`pinned ${pins.root} recursively\n`) }) @@ -171,7 +206,7 @@ describe('pin', () => { path: pins.root, recursive: true, comments: undefined - }]).returns([{ + }], defaultOptions).returns([{ cid: new CID(pins.root) }]) @@ -181,10 +216,13 @@ describe('pin', () => { }) it('recursively with timeout', async () => { - ipfs.pin.add.withArgs([pins.root], { + ipfs.pin.add.withArgs([{ + ...defaultPinOptions, + path: pins.root + }], { ...defaultOptions, timeout: 1000 - }).resolves([{ + }).returns([{ cid: new CID(pins.root) }]) From 96399fd37e7e3e27df5fb8b73900a39ef7b10ca5 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 3 Jul 2020 16:05:37 +0100 Subject: [PATCH 12/40] chore: fix tests --- packages/ipfs-core-utils/test/pins/normalise-input.spec.js | 6 +++--- packages/ipfs-http-client/test/interface.spec.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/ipfs-core-utils/test/pins/normalise-input.spec.js b/packages/ipfs-core-utils/test/pins/normalise-input.spec.js index 5f9ae32f76..fa2857ce0b 100644 --- a/packages/ipfs-core-utils/test/pins/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/pins/normalise-input.spec.js @@ -12,8 +12,8 @@ const expect = chai.expect const STRING = () => '/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn/path/to/file.txt' const PLAIN_CID = () => new CID('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') -const OBJECT_CID = () => ({ cid: new CID('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'), recursive: true, comments: 'hello world' }) -const OBJECT_PATH = () => ({ path: '/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn/path/to/file.txt', recursive: true, comments: 'hello world' }) +const OBJECT_CID = () => ({ cid: new CID('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'), recursive: true, metadata: { key: 'hello world' } }) +const OBJECT_PATH = () => ({ path: '/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn/path/to/file.txt', recursive: true, metadata: { key: 'hello world' } }) async function verifyNormalisation (input, withOptions) { const result = await all(normalise(input)) @@ -23,7 +23,7 @@ async function verifyNormalisation (input, withOptions) { if (withOptions) { expect(result[0]).to.have.property('recursive', true) - expect(result[0]).to.have.property('comments', 'hello world') + expect(result[0]).to.have.deep.property('metadata', { key: 'hello world' }) } } diff --git a/packages/ipfs-http-client/test/interface.spec.js b/packages/ipfs-http-client/test/interface.spec.js index e54358e8b0..1c1ed7c3b0 100644 --- a/packages/ipfs-http-client/test/interface.spec.js +++ b/packages/ipfs-http-client/test/interface.spec.js @@ -569,7 +569,7 @@ describe('interface-ipfs-core tests', () => { tests.pin(commonFactory, { skip: [ { - name: 'should list pins with comments', + name: 'should list pins with metadata', reason: 'not implemented in go-ipfs' } ] From d89a3307daf9e8d55bab3e05ba5fb0e47cc0739f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 21 Jul 2020 19:01:02 +0100 Subject: [PATCH 13/40] chore: split pin.add and pin.rm into pin.addAll and pin.rmAll --- docs/core-api/PIN.md | 102 ++++++++++-- examples/custom-ipfs-repo/package.json | 2 +- packages/interface-ipfs-core/package.json | 1 + packages/interface-ipfs-core/src/block/rm.js | 2 +- .../interface-ipfs-core/src/pin/add-all.js | 128 ++++++++++++++++ packages/interface-ipfs-core/src/pin/add.js | 145 +++++------------- packages/interface-ipfs-core/src/pin/index.js | 4 +- packages/interface-ipfs-core/src/pin/ls.js | 23 ++- .../interface-ipfs-core/src/pin/rm-all.js | 56 +++++++ packages/interface-ipfs-core/src/pin/rm.js | 38 ++--- packages/interface-ipfs-core/src/pin/utils.js | 17 +- .../interface-ipfs-core/src/refs-local.js | 2 +- packages/interface-ipfs-core/src/repo/gc.js | 2 +- packages/ipfs-http-client/src/pin/add-all.js | 36 +++++ packages/ipfs-http-client/src/pin/add.js | 42 ++--- packages/ipfs-http-client/src/pin/index.js | 4 +- packages/ipfs-http-client/src/pin/rm-all.js | 38 +++++ packages/ipfs-http-client/src/pin/rm.js | 44 ++---- packages/ipfs/package.json | 2 +- packages/ipfs/src/cli/commands/pin/add.js | 2 +- packages/ipfs/src/cli/commands/pin/rm.js | 2 +- .../ipfs/src/core/components/add-all/index.js | 5 +- packages/ipfs/src/core/components/dag/put.js | 5 +- packages/ipfs/src/core/components/index.js | 4 +- packages/ipfs/src/core/components/init.js | 11 +- .../ipfs/src/core/components/pin/add-all.js | 52 +++++++ packages/ipfs/src/core/components/pin/add.js | 56 +------ .../src/core/components/pin/pin-manager.js | 67 ++++++-- .../ipfs/src/core/components/pin/rm-all.js | 46 ++++++ packages/ipfs/src/core/components/pin/rm.js | 52 ++----- packages/ipfs/src/core/components/start.js | 9 +- packages/ipfs/src/core/components/stop.js | 9 +- packages/ipfs/src/core/utils.js | 36 ++--- packages/ipfs/src/http/api/resources/dag.js | 5 - packages/ipfs/src/http/api/resources/pin.js | 4 +- packages/ipfs/test/cli/pin.js | 30 ++-- packages/ipfs/test/core/gc.spec.js | 9 +- packages/ipfs/test/http-api/inject/pin.js | 70 +++++---- 38 files changed, 729 insertions(+), 433 deletions(-) create mode 100644 packages/interface-ipfs-core/src/pin/add-all.js create mode 100644 packages/interface-ipfs-core/src/pin/rm-all.js create mode 100644 packages/ipfs-http-client/src/pin/add-all.js create mode 100644 packages/ipfs-http-client/src/pin/rm-all.js create mode 100644 packages/ipfs/src/core/components/pin/add-all.js create mode 100644 packages/ipfs/src/core/components/pin/rm-all.js diff --git a/docs/core-api/PIN.md b/docs/core-api/PIN.md index 7775175c9b..fcc0571698 100644 --- a/docs/core-api/PIN.md +++ b/docs/core-api/PIN.md @@ -1,22 +1,32 @@ # Pin API -- [`ipfs.pin.add(cid, [options])`](#ipfspinaddcid-options) +- [`ipfs.pin.add(ipfsPath, [options])`](#ipfspinaddipfspath-options) - [Parameters](#parameters) - [Options](#options) - [Returns](#returns) - [Example](#example) -- [`ipfs.pin.ls([options])`](#ipfspinlsoptions) +- [`ipfs.pin.addAll(source, [options])`](#ipfspinaddallsource-options) - [Parameters](#parameters-1) - [Options](#options-1) - [Returns](#returns-1) - [Example](#example-1) -- [`ipfs.pin.rm(cid, [options])`](#ipfspinrmcid-options) +- [`ipfs.pin.ls([options])`](#ipfspinlsoptions) - [Parameters](#parameters-2) - [Options](#options-2) - [Returns](#returns-2) - [Example](#example-2) - -## `ipfs.pin.add(source, [options])` +- [`ipfs.pin.rm(ipfsPath, [options])`](#ipfspinrmipfspath-options) + - [Parameters](#parameters-3) + - [Options](#options-3) + - [Returns](#returns-3) + - [Example](#example-3) +- [`ipfs.pin.rmAll(source, [options])`](#ipfspinrmallsource-options) + - [Parameters](#parameters-4) + - [Options](#options-4) + - [Returns](#returns-4) + - [Example](#example-4) + +## `ipfs.pin.add(ipfsPath, [options])` > Adds an IPFS object to the pinset and also stores it to the IPFS repo. pinset is the set of hashes currently pinned (not gc'able) @@ -24,7 +34,7 @@ | Name | Type | Description | | ---- | ---- | ----------- | -| source | [CID][], String or `AsyncIterable<{ cid: CID, path: String, recursive: Boolean, comments: String }>` | One or more CIDs or IPFS Paths to pin in your repo | +| source | [CID][] or String | A CID or IPFS Path to pin in your repo | ### Options @@ -38,6 +48,42 @@ An optional object which may have the following keys: ### Returns +| Type | Description | +| -------- | -------- | +| [CID[] | The CIDs that was pinned | + +### Example + +```JavaScript +const cid of ipfs.pin.add(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')) +console.log(cid) +// Logs: +// CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') +``` + +A great source of [examples][] can be found in the tests for this API. + +## `ipfs.pin.addAll(source, [options])` + +> Adds multiple IPFS objects to the pinset and also stores it to the IPFS repo. pinset is the set of hashes currently pinned (not gc'able) + +### Parameters + +| Name | Type | Description | +| ---- | ---- | ----------- | +| source | `AsyncIterable<{ cid: CID, path: String, recursive: Boolean, comments: String }>` | One or more CIDs or IPFS Paths to pin in your repo | + +### Options + +An optional object which may have the following keys: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| timeout | `Number` | `undefined` | A timeout in ms | +| signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | + +### Returns + | Type | Description | | -------- | -------- | | `AsyncIterable` | An async iterable that yields the CIDs that were pinned | @@ -53,7 +99,7 @@ Each yielded object has the form: ### Example ```JavaScript -for await (const cid of ipfs.pin.add(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) { +for await (const cid of ipfs.pin.addAll(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) { console.log(cid) } // Logs: @@ -112,7 +158,7 @@ for await (const { cid, type } of ipfs.pin.ls({ A great source of [examples][] can be found in the tests for this API. -## `ipfs.pin.rm(source, [options])` +## `ipfs.pin.rm(ipfsPath, [options])` > Unpin this block from your repo @@ -120,7 +166,7 @@ A great source of [examples][] can be found in the tests for this API. | Name | Type | Description | | ---- | ---- | ----------- | -| source | [CID][], String or `AsyncIterable<{ cid: CID, path: String, recursive: Boolean }>` | Unpin this CID | +| ipfsPath | [CID][] of String | Unpin this CID or IPFS Path | ### Options @@ -134,6 +180,42 @@ An optional object which may have the following keys: ### Returns +| Type | Description | +| -------- | -------- | +| [CID][] | The CIDs that was unpinned | + +### Example + +```JavaScript +const cid of ipfs.pin.rm(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')) +console.log(cid) +// prints the CID that was unpinned +// CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') +``` + +A great source of [examples][] can be found in the tests for this API. + +## `ipfs.pin.rmAll(source, [options])` + +> Unpin one or more blocks from your repo + +### Parameters + +| Name | Type | Description | +| ---- | ---- | ----------- | +| source | [CID][], String or `AsyncIterable<{ cid: CID, path: String, recursive: Boolean }>` | Unpin this CID | + +### Options + +An optional object which may have the following keys: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| timeout | `Number` | `undefined` | A timeout in ms | +| signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | + +### Returns + | Type | Description | | -------- | -------- | | `AsyncIterable` | An async iterable that yields the CIDs that were unpinned | @@ -141,7 +223,7 @@ An optional object which may have the following keys: ### Example ```JavaScript -for await (const cid of ipfs.pin.rm(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) { +for await (const cid of ipfs.pin.rmAll(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) { console.log(cid) } // prints the CIDs that were unpinned diff --git a/examples/custom-ipfs-repo/package.json b/examples/custom-ipfs-repo/package.json index c45306c960..14a2eabfb9 100644 --- a/examples/custom-ipfs-repo/package.json +++ b/examples/custom-ipfs-repo/package.json @@ -12,7 +12,7 @@ "dependencies": { "datastore-fs": "^1.1.0", "ipfs": "^0.48.0", - "ipfs-repo": "github:ipfs/js-ipfs-repo#store-pins-in-datastore", + "ipfs-repo": "^5.0.0", "it-all": "^1.0.1" }, "devDependencies": { diff --git a/packages/interface-ipfs-core/package.json b/packages/interface-ipfs-core/package.json index a9c6ede7e1..e67f07650f 100644 --- a/packages/interface-ipfs-core/package.json +++ b/packages/interface-ipfs-core/package.json @@ -51,6 +51,7 @@ "it-concat": "^1.0.0", "it-drain": "^1.0.1", "it-last": "^1.0.1", + "it-map": "^1.0.2", "it-pushable": "^1.3.1", "multiaddr": "^7.4.3", "multibase": "^1.0.1", diff --git a/packages/interface-ipfs-core/src/block/rm.js b/packages/interface-ipfs-core/src/block/rm.js index b77cda31b0..1e35c0f5da 100644 --- a/packages/interface-ipfs-core/src/block/rm.js +++ b/packages/interface-ipfs-core/src/block/rm.js @@ -152,7 +152,7 @@ module.exports = (common, options) => { format: 'raw', hashAlg: 'sha2-256' }) - await drain(ipfs.pin.add(cid)) + await ipfs.pin.add(cid) const result = await last(ipfs.block.rm(cid)) diff --git a/packages/interface-ipfs-core/src/pin/add-all.js b/packages/interface-ipfs-core/src/pin/add-all.js new file mode 100644 index 0000000000..138b9e2f9b --- /dev/null +++ b/packages/interface-ipfs-core/src/pin/add-all.js @@ -0,0 +1,128 @@ +/* eslint-env mocha */ +'use strict' + +const { fixtures, clearPins } = require('./utils') +const { getDescribe, getIt, expect } = require('../utils/mocha') +const all = require('it-all') +const drain = require('it-drain') +const testTimeout = require('../utils/test-timeout') +const CID = require('cids') + +/** @typedef { import("ipfsd-ctl/src/factory") } Factory */ +/** + * @param {Factory} common + * @param {Object} options + */ +module.exports = (common, options) => { + const describe = getDescribe(options) + const it = getIt(options) + + describe('.pin.addAll', function () { + this.timeout(50 * 1000) + + let ipfs + before(async () => { + ipfs = (await common.spawn()).api + + await drain( + ipfs.addAll( + fixtures.files.map(file => ({ content: file.data })), { + pin: false + } + ) + ) + + await drain( + ipfs.addAll(fixtures.directory.files.map( + file => ({ + path: file.path, + content: file.data + }) + ), { + pin: false + }) + ) + }) + + after(() => common.clean()) + + beforeEach(() => { + return clearPins(ipfs) + }) + + async function testAddPinInput (source) { + const pinset = await all(ipfs.pin.addAll(source)) + + expect(pinset).to.have.deep.members([ + fixtures.files[0].cid, + fixtures.files[1].cid + ]) + } + + it('should add an array of CIDs', () => { + return testAddPinInput([ + fixtures.files[0].cid, + fixtures.files[1].cid + ]) + }) + + it('should add a generator of CIDs', () => { + return testAddPinInput(function * () { + yield fixtures.files[0].cid + yield fixtures.files[1].cid + }()) + }) + + it('should add an async generator of CIDs', () => { + return testAddPinInput(async function * () { // eslint-disable-line require-await + yield fixtures.files[0].cid + yield fixtures.files[1].cid + }()) + }) + + it('should add an array of pins with options', () => { + return testAddPinInput([ + { + cid: fixtures.files[0].cid, + recursive: false + }, + { + cid: fixtures.files[1].cid, + recursive: true + } + ]) + }) + + it('should add a generator of pins with options', () => { + return testAddPinInput(function * () { + yield { + cid: fixtures.files[0].cid, + recursive: false + } + yield { + cid: fixtures.files[1].cid, + recursive: true + } + }()) + }) + + it('should add an async generator of pins with options', () => { + return testAddPinInput(async function * () { // eslint-disable-line require-await + yield { + cid: fixtures.files[0].cid, + recursive: false + } + yield { + cid: fixtures.files[1].cid, + recursive: true + } + }()) + }) + + it('should respect timeout option when pinning a block', () => { + return testTimeout(() => ipfs.pin.addAll([new CID('Qmd7qZS4T7xXtsNFdRoK1trfMs5zU94EpokQ9WFtxdPxsZ')], { + timeout: 1 + })) + }) + }) +} diff --git a/packages/interface-ipfs-core/src/pin/add.js b/packages/interface-ipfs-core/src/pin/add.js index ba987e9646..6010a24151 100644 --- a/packages/interface-ipfs-core/src/pin/add.js +++ b/packages/interface-ipfs-core/src/pin/add.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { fixtures, clearPins, expectPinned, pinTypes } = require('./utils') +const { fixtures, clearPins, expectPinned, expectNotPinned, pinTypes } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const all = require('it-all') const drain = require('it-drain') @@ -27,12 +27,16 @@ module.exports = (common, options) => { before(async () => { ipfs = (await common.spawn()).api - await Promise.all(fixtures.files.map(file => { - return ipfs.add(file.data, { pin: false }) - })) + await drain( + ipfs.addAll( + fixtures.files.map(file => ({ content: file.data })), { + pin: false + } + ) + ) - await all( - ipfs.add(fixtures.directory.files.map( + await drain( + ipfs.addAll(fixtures.directory.files.map( file => ({ path: file.path, content: file.data @@ -49,90 +53,20 @@ module.exports = (common, options) => { return clearPins(ipfs) }) - async function testAddInput (source) { - const pinset = await all(ipfs.pin.add(source)) - - expect(pinset).to.have.deep.members([ - fixtures.files[0].cid, - fixtures.files[1].cid - ]) - } - it('should add a CID and return the added CID', async () => { - const pinset = await all(ipfs.pin.add(fixtures.files[0].cid)) - expect(pinset).to.deep.include(fixtures.files[0].cid) + const cid = await ipfs.pin.add(fixtures.files[0].cid) + expect(cid).to.deep.equal(fixtures.files[0].cid) }) it('should add a pin with options and return the added CID', async () => { - const pinset = await all(ipfs.pin.add({ - cid: fixtures.files[0].cid, + const cid = await ipfs.pin.add(fixtures.files[0].cid, { recursive: false - })) - expect(pinset).to.deep.include(fixtures.files[0].cid) - }) - - it('should add an array of CIDs', () => { - return testAddInput([ - fixtures.files[0].cid, - fixtures.files[1].cid - ]) - }) - - it('should add a generator of CIDs', () => { - return testAddInput(function * () { - yield fixtures.files[0].cid - yield fixtures.files[1].cid - }()) - }) - - it('should add an async generator of CIDs', () => { - return testAddInput(async function * () { // eslint-disable-line require-await - yield fixtures.files[0].cid - yield fixtures.files[1].cid - }()) - }) - - it('should add an array of pins with options', () => { - return testAddInput([ - { - cid: fixtures.files[0].cid, - recursive: false - }, - { - cid: fixtures.files[1].cid, - recursive: true - } - ]) - }) - - it('should add a generator of pins with options', () => { - return testAddInput(function * () { - yield { - cid: fixtures.files[0].cid, - recursive: false - } - yield { - cid: fixtures.files[1].cid, - recursive: true - } - }()) - }) - - it('should add an async generator of pins with options', () => { - return testAddInput(async function * () { // eslint-disable-line require-await - yield { - cid: fixtures.files[0].cid, - recursive: false - } - yield { - cid: fixtures.files[1].cid, - recursive: true - } - }()) + }) + expect(cid).to.deep.equal(fixtures.files[0].cid) }) it('should add recursively', async () => { - await drain(ipfs.pin.add(fixtures.directory.cid)) + await ipfs.pin.add(fixtures.directory.cid) await expectPinned(ipfs, fixtures.directory.cid, pinTypes.recursive) const pinChecks = Object.values(fixtures.directory.files).map(file => expectPinned(ipfs, file.cid)) @@ -140,35 +74,30 @@ module.exports = (common, options) => { }) it('should add directly', async () => { - await drain(ipfs.pin.add({ - cid: fixtures.directory.cid, + await ipfs.pin.add(fixtures.directory.cid, { recursive: false - })) - await Promise.all([ - expectPinned(ipfs, fixtures.directory.cid, pinTypes.direct), - expectPinned(ipfs, fixtures.directory.files[0].cid, false) - ]) + }) + + await expectPinned(ipfs, fixtures.directory.cid, pinTypes.direct) + await expectNotPinned(ipfs, fixtures.directory.files[0].cid) }) it('should recursively pin parent of direct pin', async () => { - await drain(ipfs.pin.add({ - cid: fixtures.directory.files[0].cid, + await ipfs.pin.add(fixtures.directory.files[0].cid, { recursive: false - })) - await drain(ipfs.pin.add(fixtures.directory.cid)) - await Promise.all([ - // file is pinned both directly and indirectly o.O - expectPinned(ipfs, fixtures.directory.files[0].cid, pinTypes.direct), - expectPinned(ipfs, fixtures.directory.files[0].cid, pinTypes.indirect) - ]) + }) + await ipfs.pin.add(fixtures.directory.cid) + + // file is pinned both directly and indirectly o.O + await expectPinned(ipfs, fixtures.directory.files[0].cid, pinTypes.direct) + await expectPinned(ipfs, fixtures.directory.files[0].cid, pinTypes.indirect) }) it('should fail to directly pin a recursive pin', async () => { - await drain(ipfs.pin.add(fixtures.directory.cid)) - return expect(drain(ipfs.pin.add({ - cid: fixtures.directory.cid, + await ipfs.pin.add(fixtures.directory.cid) + return expect(ipfs.pin.add(fixtures.directory.cid, { recursive: false - }))) + })) .to.eventually.be.rejected() .with(/already pinned recursively/) }) @@ -177,7 +106,7 @@ module.exports = (common, options) => { this.slow(3 * 1000) this.timeout(5 * 1000) const falseHash = `${`${fixtures.directory.cid}`.slice(0, -2)}ss` - return expect(drain(ipfs.pin.add(falseHash, { timeout: '2s' }))) + return expect(ipfs.pin.add(falseHash, { timeout: '2s' })) .to.eventually.be.rejected() // TODO: http api TimeoutErrors do not have this property // .with.a.property('code').that.equals('ERR_TIMEOUT') @@ -188,7 +117,7 @@ module.exports = (common, options) => { this.timeout(5 * 1000) await all(ipfs.block.rm(fixtures.directory.files[0].cid)) - await expect(drain(ipfs.pin.add(fixtures.directory.cid, { timeout: '2s' }))) + await expect(ipfs.pin.add(fixtures.directory.cid, { timeout: '2s' })) .to.eventually.be.rejected() }) @@ -198,7 +127,7 @@ module.exports = (common, options) => { hashAlg: 'sha2-256' }) - await drain(ipfs.pin.add(cid)) + await ipfs.pin.add(cid) const pins = await all(ipfs.pin.ls()) @@ -214,7 +143,7 @@ module.exports = (common, options) => { hashAlg: 'sha2-256' }) - await drain(ipfs.pin.add(cid)) + await ipfs.pin.add(cid) const pins = await all(ipfs.pin.ls()) @@ -236,9 +165,9 @@ module.exports = (common, options) => { hashAlg: 'sha2-256' }) - await drain(ipfs.pin.add(parent, { + await ipfs.pin.add(parent, { recursive: true - })) + }) const pins = await all(ipfs.pin.ls()) diff --git a/packages/interface-ipfs-core/src/pin/index.js b/packages/interface-ipfs-core/src/pin/index.js index 15f55419d3..c4c48c17ae 100644 --- a/packages/interface-ipfs-core/src/pin/index.js +++ b/packages/interface-ipfs-core/src/pin/index.js @@ -3,8 +3,10 @@ const { createSuite } = require('../utils/suite') const tests = { add: require('./add'), + addAll: require('./add-all'), ls: require('./ls'), - rm: require('./rm') + rm: require('./rm'), + rmAll: require('./rm-all') } module.exports = createSuite(tests) diff --git a/packages/interface-ipfs-core/src/pin/ls.js b/packages/interface-ipfs-core/src/pin/ls.js index 76ab85f2b8..111d84459b 100644 --- a/packages/interface-ipfs-core/src/pin/ls.js +++ b/packages/interface-ipfs-core/src/pin/ls.js @@ -4,8 +4,6 @@ const { fixtures } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const all = require('it-all') -const drain = require('it-drain') -const last = require('it-last') const testTimeout = require('../utils/test-timeout') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ @@ -27,19 +25,19 @@ module.exports = (common, options) => { // two files wrapped in directories, only root CID pinned recursively const dir = fixtures.directory.files.map((file) => ({ path: file.path, content: file.data })) await all(ipfs.addAll(dir, { pin: false, cidVersion: 0 })) - await drain(ipfs.pin.add(fixtures.directory.cid, { recursive: true })) + await ipfs.pin.add(fixtures.directory.cid, { recursive: true }) // a file (CID pinned recursively) await ipfs.add(fixtures.files[0].data, { pin: false, cidVersion: 0 }) - await drain(ipfs.pin.add(fixtures.files[0].cid, { recursive: true })) + await ipfs.pin.add(fixtures.files[0].cid, { recursive: true }) // a single CID (pinned directly) await ipfs.add(fixtures.files[1].data, { pin: false, cidVersion: 0 }) - await drain(ipfs.pin.add(fixtures.files[1].cid, { recursive: false })) + await ipfs.pin.add(fixtures.files[1].cid, { recursive: false }) }) after(() => common.clean()) it('should respect timeout option when listing pins', () => { - return testTimeout(() => ipfs.pin.ls(undefined, { + return testTimeout(() => ipfs.pin.ls({ timeout: 1 })) }) @@ -180,8 +178,8 @@ module.exports = (common, options) => { paths: [fixtures.files[0].cid, fixtures.files[1].cid] })) const cids = pinset.map(p => p.cid.toString()) - expect(cids).to.include(fixtures.files[0].cid) - expect(cids).to.include(fixtures.files[1].cid) + expect(cids).to.include(fixtures.files[0].cid.toString()) + expect(cids).to.include(fixtures.files[1].cid.toString()) }) it('should throw error for invalid non-string pin type option', () => { @@ -199,9 +197,9 @@ module.exports = (common, options) => { }) it('should list pins with metadata', async () => { - const { cid } = await last(ipfs.add(`data-${Math.random()}`, { + const { cid } = await ipfs.add(`data-${Math.random()}`, { pin: false - })) + }) const metadata = { key: 'value', @@ -215,11 +213,10 @@ module.exports = (common, options) => { } } - await drain(ipfs.pin.add({ - cid: cid, + await ipfs.pin.add(cid, { recursive: false, metadata - })) + }) const pinset = await all(ipfs.pin.ls({ paths: cid diff --git a/packages/interface-ipfs-core/src/pin/rm-all.js b/packages/interface-ipfs-core/src/pin/rm-all.js new file mode 100644 index 0000000000..b2f319ac7e --- /dev/null +++ b/packages/interface-ipfs-core/src/pin/rm-all.js @@ -0,0 +1,56 @@ +/* eslint-env mocha */ +'use strict' + +const { fixtures, clearPins } = require('./utils') +const { getDescribe, getIt, expect } = require('../utils/mocha') +const all = require('it-all') +const drain = require('it-drain') +const testTimeout = require('../utils/test-timeout') + +/** @typedef { import("ipfsd-ctl/src/factory") } Factory */ +/** + * @param {Factory} common + * @param {Object} options + */ +module.exports = (common, options) => { + const describe = getDescribe(options) + const it = getIt(options) + + describe('.pin.rmAll', function () { + this.timeout(50 * 1000) + + let ipfs + beforeEach(async () => { + ipfs = (await common.spawn()).api + + const dir = fixtures.directory.files.map((file) => ({ path: file.path, content: file.data })) + await all(ipfs.addAll(dir, { pin: false, cidVersion: 0 })) + + await ipfs.add(fixtures.files[0].data, { pin: false }) + await ipfs.add(fixtures.files[1].data, { pin: false }) + }) + + after(() => common.clean()) + + beforeEach(() => { + return clearPins(ipfs) + }) + + it('should respect timeout option when unpinning a block', async () => { + await ipfs.pin.add(fixtures.files[0].cid, { recursive: true }) + + await testTimeout(() => ipfs.pin.rmAll([fixtures.files[0].cid], { + recursive: true, + timeout: 1 + })) + }) + + it('should pipe the output of ls to rm', async () => { + await ipfs.pin.add(fixtures.directory.cid) + + await drain(ipfs.pin.rmAll(ipfs.pin.ls({ type: 'recursive' }))) + + await expect(all(ipfs.pin.ls())).to.eventually.have.lengthOf(0) + }) + }) +} diff --git a/packages/interface-ipfs-core/src/pin/rm.js b/packages/interface-ipfs-core/src/pin/rm.js index 13c6acbccd..7ae804518f 100644 --- a/packages/interface-ipfs-core/src/pin/rm.js +++ b/packages/interface-ipfs-core/src/pin/rm.js @@ -4,7 +4,6 @@ const { fixtures, expectPinned, clearPins } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const all = require('it-all') -const drain = require('it-drain') const testTimeout = require('../utils/test-timeout') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ @@ -22,10 +21,11 @@ module.exports = (common, options) => { let ipfs beforeEach(async () => { ipfs = (await common.spawn()).api + const dir = fixtures.directory.files.map((file) => ({ path: file.path, content: file.data })) + await all(ipfs.addAll(dir, { pin: false, cidVersion: 0 })) + await ipfs.add(fixtures.files[0].data, { pin: false }) - await drain(ipfs.pin.add(fixtures.files[0].cid, { recursive: true })) await ipfs.add(fixtures.files[1].data, { pin: false }) - await drain(ipfs.pin.add(fixtures.files[1].cid, { recursive: false })) }) after(() => common.clean()) @@ -34,18 +34,20 @@ module.exports = (common, options) => { return clearPins(ipfs) }) - it('should respect timeout option when unpinning a block', () => { - return testTimeout(() => ipfs.pin.rm(fixtures.files[0].cid, { + it('should respect timeout option when unpinning a block', async () => { + await ipfs.pin.add(fixtures.files[0].cid, { recursive: true }) + + await testTimeout(() => ipfs.pin.rm(fixtures.files[0].cid, { recursive: true, timeout: 1 })) }) it('should remove a recursive pin', async () => { - await drain(ipfs.pin.add(fixtures.directory.cid)) + await ipfs.pin.add(fixtures.directory.cid) - const removedPinset = await all(ipfs.pin.rm(fixtures.directory.cid, { recursive: true })) - expect(removedPinset).to.deep.equal([fixtures.directory.cid]) + const unpinnedCid = await ipfs.pin.rm(fixtures.directory.cid, { recursive: true }) + expect(unpinnedCid).to.deep.equal(fixtures.directory.cid) const pinset = await all(ipfs.pin.ls({ type: 'recursive' })) expect(pinset).to.not.deep.include({ @@ -55,36 +57,28 @@ module.exports = (common, options) => { }) it('should remove a direct pin', async () => { - await drain(ipfs.pin.add(fixtures.directory.cid, { recursive: false })) + await ipfs.pin.add(fixtures.directory.cid, { recursive: false }) - const removedPinset = await all(ipfs.pin.rm(fixtures.directory.cid, { recursive: false })) - expect(removedPinset).to.deep.equal([fixtures.directory.cid]) + const unpinnedCid = await ipfs.pin.rm(fixtures.directory.cid, { recursive: false }) + expect(unpinnedCid).to.deep.equal(fixtures.directory.cid) const pinset = await all(ipfs.pin.ls({ type: 'direct' })) expect(pinset.map(p => p.cid)).to.not.deep.include(fixtures.directory.cid) }) it('should fail to remove an indirect pin', async () => { - await drain(ipfs.pin.add(fixtures.directory.cid)) + await ipfs.pin.add(fixtures.directory.cid) - await expect(drain(ipfs.pin.rm(fixtures.directory.files[0].cid))) + await expect(ipfs.pin.rm(fixtures.directory.files[0].cid)) .to.eventually.be.rejected() .with(/is pinned indirectly under/) await expectPinned(ipfs, fixtures.directory.files[0].cid) }) it('should fail when an item is not pinned', async () => { - await expect(drain(ipfs.pin.rm(fixtures.directory.cid))) + await expect(ipfs.pin.rm(fixtures.directory.cid)) .to.eventually.be.rejected() .with(/is not pinned/) }) - - it('should pipe the output of ls to rm', async () => { - await drain(ipfs.pin.add(fixtures.directory.cid)) - - await drain(ipfs.pin.rm(ipfs.pin.ls({ type: 'recursive' }))) - - await expect(all(ipfs.pin.ls())).to.eventually.have.lengthOf(0) - }) }) } diff --git a/packages/interface-ipfs-core/src/pin/utils.js b/packages/interface-ipfs-core/src/pin/utils.js index a1ad57ac8e..002f2fe26d 100644 --- a/packages/interface-ipfs-core/src/pin/utils.js +++ b/packages/interface-ipfs-core/src/pin/utils.js @@ -4,6 +4,7 @@ const { expect } = require('../utils/mocha') const loadFixture = require('aegir/fixtures') const CID = require('cids') const drain = require('it-drain') +const map = require('it-map') const pinTypes = { direct: 'direct', @@ -36,13 +37,8 @@ const fixtures = Object.freeze({ }) const clearPins = async (ipfs) => { - for await (const { cid } of ipfs.pin.ls(null, { type: pinTypes.recursive })) { - await drain(ipfs.pin.rm(cid)) - } - - for await (const { cid } of ipfs.pin.ls(null, { type: pinTypes.direct })) { - await drain(ipfs.pin.rm(cid)) - } + await drain(ipfs.pin.rmAll(map(ipfs.pin.ls({ type: pinTypes.recursive }), ({ cid }) => cid))) + await drain(ipfs.pin.rmAll(map(ipfs.pin.ls({ type: pinTypes.direct }), ({ cid }) => cid))) } const expectPinned = async (ipfs, cid, type = pinTypes.all, pinned = true) => { @@ -55,9 +51,13 @@ const expectPinned = async (ipfs, cid, type = pinTypes.all, pinned = true) => { expect(result).to.eql(pinned) } +const expectNotPinned = (ipfs, cid, type = pinTypes.all) => { + return expectPinned(ipfs, cid, type, false) +} + async function isPinnedWithType (ipfs, cid, type) { try { - for await (const _ of ipfs.pin.ls(cid, { type })) { // eslint-disable-line no-unused-vars + for await (const _ of ipfs.pin.ls({ paths: cid, type })) { // eslint-disable-line no-unused-vars return true } return false @@ -70,6 +70,7 @@ module.exports = { fixtures, clearPins, expectPinned, + expectNotPinned, isPinnedWithType, pinTypes } diff --git a/packages/interface-ipfs-core/src/refs-local.js b/packages/interface-ipfs-core/src/refs-local.js index 2c59e18dd4..ff8ed0eeb9 100644 --- a/packages/interface-ipfs-core/src/refs-local.js +++ b/packages/interface-ipfs-core/src/refs-local.js @@ -49,7 +49,7 @@ module.exports = (common, options) => { const imported = await all(importer(dirs, ipfs.block)) // otherwise go-ipfs doesn't show them in the local refs - await drain(ipfs.pin.add(imported.map(i => i.cid))) + await ipfs.pin.add(imported.map(i => i.cid)) const refs = await all(ipfs.refs.local()) const cids = refs.map(r => r.ref) diff --git a/packages/interface-ipfs-core/src/repo/gc.js b/packages/interface-ipfs-core/src/repo/gc.js index a0de56ff79..8cf0b1e7db 100644 --- a/packages/interface-ipfs-core/src/repo/gc.js +++ b/packages/interface-ipfs-core/src/repo/gc.js @@ -202,7 +202,7 @@ module.exports = (common, options) => { expect(refsAfterAdd.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash) // Recursively pin the object - await drain(ipfs.pin.add(objCid, { recursive: true })) + await ipfs.pin.add(objCid, { recursive: true }) // The data should now be indirectly pinned const pins = await all(ipfs.pin.ls()) diff --git a/packages/ipfs-http-client/src/pin/add-all.js b/packages/ipfs-http-client/src/pin/add-all.js new file mode 100644 index 0000000000..e49f386d4d --- /dev/null +++ b/packages/ipfs-http-client/src/pin/add-all.js @@ -0,0 +1,36 @@ +'use strict' + +const CID = require('cids') +const configure = require('../lib/configure') +const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const toUrlSearchParams = require('../lib/to-url-search-params') + +module.exports = configure(api => { + return async function * addAll (source, options = {}) { + for await (const { path, recursive, metadata } of normaliseInput(source)) { + const res = await api.post('pin/add', { + timeout: options.timeout, + signal: options.signal, + searchParams: toUrlSearchParams({ + ...options, + arg: path, + recursive, + metadata: metadata ? JSON.stringify(metadata) : undefined, + stream: true + }), + headers: options.headers + }) + + for await (const pin of res.ndjson()) { + if (pin.Pins) { // non-streaming response + for (const cid of pin.Pins) { + yield new CID(cid) + } + continue + } + + yield new CID(pin) + } + } + } +}) diff --git a/packages/ipfs-http-client/src/pin/add.js b/packages/ipfs-http-client/src/pin/add.js index dc556a51ef..7303010169 100644 --- a/packages/ipfs-http-client/src/pin/add.js +++ b/packages/ipfs-http-client/src/pin/add.js @@ -1,36 +1,18 @@ 'use strict' -const CID = require('cids') +const addAll = require('./add-all') +const last = require('it-last') const configure = require('../lib/configure') -const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') -const toUrlSearchParams = require('../lib/to-url-search-params') -module.exports = configure(api => { - return async function * (source, options = {}) { - for await (const { path, recursive, metadata } of normaliseInput(source)) { - const res = await api.post('pin/add', { - timeout: options.timeout, - signal: options.signal, - searchParams: toUrlSearchParams({ - arg: path, - recursive, - metadata: metadata ? JSON.stringify(metadata) : undefined, - ...options, - stream: true - }), - headers: options.headers - }) +module.exports = (options) => { + const all = addAll(options) - for await (const pin of res.ndjson()) { - if (pin.Pins) { // non-streaming response - for (const cid of pin.Pins) { - yield new CID(cid) - } - continue - } - - yield new CID(pin) - } + return configure(() => { + return async function add (path, options = {}) { // eslint-disable-line require-await + return last(all({ + path, + ...options + }, options)) } - } -}) + })(options) +} diff --git a/packages/ipfs-http-client/src/pin/index.js b/packages/ipfs-http-client/src/pin/index.js index ad43057ec5..39f7b5ecc8 100644 --- a/packages/ipfs-http-client/src/pin/index.js +++ b/packages/ipfs-http-client/src/pin/index.js @@ -2,6 +2,8 @@ module.exports = config => ({ add: require('./add')(config), + addAll: require('./add-all')(config), + ls: require('./ls')(config), rm: require('./rm')(config), - ls: require('./ls')(config) + rmAll: require('./rm-all')(config) }) diff --git a/packages/ipfs-http-client/src/pin/rm-all.js b/packages/ipfs-http-client/src/pin/rm-all.js new file mode 100644 index 0000000000..4e64456215 --- /dev/null +++ b/packages/ipfs-http-client/src/pin/rm-all.js @@ -0,0 +1,38 @@ +'use strict' + +const CID = require('cids') +const configure = require('../lib/configure') +const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const toUrlSearchParams = require('../lib/to-url-search-params') + +module.exports = configure(api => { + return async function * rmAll (source, options = {}) { + options = options || {} + + for await (const { path, recursive } of normaliseInput(source)) { + const searchParams = new URLSearchParams(options.searchParams) + searchParams.append('arg', `${path}`) + + if (recursive != null) searchParams.set('recursive', recursive) + + const res = await api.post('pin/rm', { + timeout: options.timeout, + signal: options.signal, + headers: options.headers, + searchParams: toUrlSearchParams({ + ...options, + arg: `${path}`, + recursive + }) + }) + + for await (const pin of res.ndjson()) { + if (pin.Pins) { // non-streaming response + yield * pin.Pins.map(cid => new CID(cid)) + continue + } + yield new CID(pin) + } + } + } +}) diff --git a/packages/ipfs-http-client/src/pin/rm.js b/packages/ipfs-http-client/src/pin/rm.js index e2d84babb2..34f3daf067 100644 --- a/packages/ipfs-http-client/src/pin/rm.js +++ b/packages/ipfs-http-client/src/pin/rm.js @@ -1,38 +1,18 @@ 'use strict' -const CID = require('cids') +const rmAll = require('./rm-all') +const last = require('it-last') const configure = require('../lib/configure') -const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') -const toUrlSearchParams = require('../lib/to-url-search-params') -module.exports = configure(api => { - return async function * (source, options = {}) { - options = options || {} +module.exports = (options) => { + const all = rmAll(options) - for await (const { path, recursive } of normaliseInput(source)) { - const searchParams = new URLSearchParams(options.searchParams) - searchParams.append('arg', `${path}`) - - if (recursive != null) searchParams.set('recursive', recursive) - - const res = await api.post('pin/rm', { - timeout: options.timeout, - signal: options.signal, - headers: options.headers, - searchParams: toUrlSearchParams({ - ...options, - arg: `${path}`, - recursive - }) - }) - - for await (const pin of res.ndjson()) { - if (pin.Pins) { // non-streaming response - yield * pin.Pins.map(cid => new CID(cid)) - continue - } - yield new CID(pin) - } + return configure(() => { + return async function rm (path, options = {}) { // eslint-disable-line require-await + return last(all({ + path, + ...options + }, options)) } - } -}) + })(options) +} diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index c153d9f6f9..d26bdda4ea 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -97,7 +97,7 @@ "ipfs-core-utils": "^0.3.0", "ipfs-http-client": "^45.0.0", "ipfs-http-response": "^0.5.0", - "ipfs-repo": "github:ipfs/js-ipfs-repo#store-pins-in-datastore", + "ipfs-repo": "^5.0.0", "ipfs-unixfs": "^1.0.3", "ipfs-unixfs-exporter": "^2.0.2", "ipfs-unixfs-importer": "^2.0.2", diff --git a/packages/ipfs/src/cli/commands/pin/add.js b/packages/ipfs/src/cli/commands/pin/add.js index 96c68b2d83..778ee4ac9e 100644 --- a/packages/ipfs/src/cli/commands/pin/add.js +++ b/packages/ipfs/src/cli/commands/pin/add.js @@ -59,7 +59,7 @@ module.exports = { metadata = metadataJson } - for await (const res of ipfs.pin.add(ipfsPath.map(path => ({ path, recursive, metadata })), { timeout })) { + for await (const res of ipfs.pin.addAll(ipfsPath.map(path => ({ path, recursive, metadata })), { timeout })) { print(`pinned ${cidToString(res.cid, { base: cidBase })} ${type}ly`) } } diff --git a/packages/ipfs/src/cli/commands/pin/rm.js b/packages/ipfs/src/cli/commands/pin/rm.js index b297863f9c..1a8a246e30 100644 --- a/packages/ipfs/src/cli/commands/pin/rm.js +++ b/packages/ipfs/src/cli/commands/pin/rm.js @@ -30,7 +30,7 @@ module.exports = { async handler ({ ctx, ipfsPath, timeout, recursive, cidBase }) { const { ipfs, print } = ctx - for await (const res of ipfs.pin.rm(ipfsPath.map(path => ({ path, recursive })), { timeout })) { + for await (const res of ipfs.pin.rmAll(ipfsPath.map(path => ({ path, recursive })), { timeout })) { print(`unpinned ${cidToString(res.cid, { base: cidBase })}`) } } diff --git a/packages/ipfs/src/core/components/add-all/index.js b/packages/ipfs/src/core/components/add-all/index.js index 78ac93e0e8..f72c337439 100644 --- a/packages/ipfs/src/core/components/add-all/index.js +++ b/packages/ipfs/src/core/components/add-all/index.js @@ -4,7 +4,6 @@ const importer = require('ipfs-unixfs-importer') const normaliseAddInput = require('ipfs-core-utils/src/files/normalise-input') const { parseChunkerString } = require('./utils') const pipe = require('it-pipe') -const drain = require('it-drain') const { withTimeoutOption } = require('../../utils') module.exports = ({ block, gcLock, preload, pin, options: constructorOptions }) => { @@ -117,10 +116,10 @@ function pinFile (pin, opts) { if (shouldPin) { // Note: addAsyncIterator() has already taken a GC lock, so tell // pin.add() not to take a (second) GC lock - await drain(pin.add(file.cid, { + await pin.add(file.cid, { preload: false, lock: false - })) + }) } yield file diff --git a/packages/ipfs/src/core/components/dag/put.js b/packages/ipfs/src/core/components/dag/put.js index 601ca985ce..92a270ad23 100644 --- a/packages/ipfs/src/core/components/dag/put.js +++ b/packages/ipfs/src/core/components/dag/put.js @@ -2,7 +2,6 @@ const multicodec = require('multicodec') const nameToCodec = name => multicodec[name.toUpperCase().replace(/-/g, '_')] -const drain = require('it-drain') const { withTimeoutOption } = require('../../utils') module.exports = ({ ipld, pin, gcLock, preload }) => { @@ -54,9 +53,9 @@ module.exports = ({ ipld, pin, gcLock, preload }) => { }) if (options.pin) { - await drain(pin.add(cid, { + await pin.add(cid, { lock: false - })) + }) } if (options.preload !== false) { diff --git a/packages/ipfs/src/core/components/index.js b/packages/ipfs/src/core/components/index.js index c5257df661..8d2abb01b2 100644 --- a/packages/ipfs/src/core/components/index.js +++ b/packages/ipfs/src/core/components/index.js @@ -72,8 +72,10 @@ exports.object = { } exports.pin = { add: require('./pin/add'), + addAll: require('./pin/add-all'), ls: require('./pin/ls'), - rm: require('./pin/rm') + rm: require('./pin/rm'), + rmAll: require('./pin/rm-all') } exports.ping = require('./ping') exports.pubsub = require('./pubsub') diff --git a/packages/ipfs/src/core/components/init.js b/packages/ipfs/src/core/components/init.js index 76fe1f86a9..11061eb344 100644 --- a/packages/ipfs/src/core/components/init.js +++ b/packages/ipfs/src/core/components/init.js @@ -27,7 +27,6 @@ const OfflineDatastore = require('../ipns/routing/offline-datastore') const initAssets = require('../runtime/init-assets-nodejs') const PinManager = require('./pin/pin-manager') const Components = require('./') -const drain = require('it-drain') module.exports = ({ apiManager, @@ -110,11 +109,15 @@ module.exports = ({ } const pinManager = new PinManager(repo, dag) + const pinAddAll = Components.pin.addAll({ pinManager, gcLock, dag }) + const pinRmAll = Components.pin.rmAll({ pinManager, gcLock, dag }) const pin = { - add: Components.pin.add({ pinManager, gcLock, dag }), + add: Components.pin.add({ addAll: pinAddAll }), + addAll: pinAddAll, ls: Components.pin.ls({ pinManager, dag }), - rm: Components.pin.rm({ pinManager, gcLock, dag }) + rm: Components.pin.rm({ rmAll: pinRmAll }), + rmAll: pinRmAll } // FIXME: resolve this circular dependency @@ -278,7 +281,7 @@ async function addEmptyDir ({ dag, pin }) { hashAlg: multicodec.SHA2_256, preload: false }) - await drain(pin.add(cid)) + await pin.add(cid) return cid } diff --git a/packages/ipfs/src/core/components/pin/add-all.js b/packages/ipfs/src/core/components/pin/add-all.js new file mode 100644 index 0000000000..4a5559022b --- /dev/null +++ b/packages/ipfs/src/core/components/pin/add-all.js @@ -0,0 +1,52 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +'use strict' + +const { resolvePath, withTimeoutOption } = require('../../utils') +const PinManager = require('./pin-manager') +const { PinTypes } = PinManager +const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') + +module.exports = ({ pinManager, gcLock, dag }) => { + return withTimeoutOption(async function * addAll (source, options) { + options = options || {} + + const pinAdd = async function * () { + for await (const { path, recursive, metadata } of normaliseInput(source)) { + const cid = await resolvePath(dag, path) + + // verify that each hash can be pinned + const { reason } = await pinManager.isPinnedWithType(cid, [PinTypes.recursive, PinTypes.direct]) + + if (reason === 'recursive' && !recursive) { + // only disallow trying to override recursive pins + throw new Error(`${cid} already pinned recursively`) + } + + if (recursive) { + await pinManager.pinRecursively(cid, { metadata }) + } else { + await pinManager.pinDirectly(cid, { metadata }) + } + + yield cid + } + } + + // When adding a file, we take a lock that gets released after pinning + // is complete, so don't take a second lock here + const lock = Boolean(options.lock) + + if (!lock) { + yield * pinAdd() + return + } + + const release = await gcLock.readLock() + + try { + yield * pinAdd() + } finally { + release() + } + }) +} diff --git a/packages/ipfs/src/core/components/pin/add.js b/packages/ipfs/src/core/components/pin/add.js index 2b4bd40b83..4e5ee88dd4 100644 --- a/packages/ipfs/src/core/components/pin/add.js +++ b/packages/ipfs/src/core/components/pin/add.js @@ -1,52 +1,12 @@ -/* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { resolvePath, withTimeoutOption } = require('../../utils') -const PinManager = require('./pin-manager') -const { PinTypes } = PinManager -const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const last = require('it-last') -module.exports = ({ pinManager, gcLock, dag }) => { - return withTimeoutOption(async function * add (source, options) { - options = options || {} - - const pinAdd = async function * () { - for await (const { path, recursive, metadata } of normaliseInput(source)) { - const cid = await resolvePath(dag, path) - - // verify that each hash can be pinned - const { reason } = await pinManager.isPinnedWithType(cid, [PinTypes.recursive, PinTypes.direct]) - - if (reason === 'recursive' && !recursive) { - // only disallow trying to override recursive pins - throw new Error(`${cid} already pinned recursively`) - } - - if (recursive) { - await pinManager.pinRecursively(cid, { metadata }) - } else { - await pinManager.pinDirectly(cid, { metadata }) - } - - yield cid - } - } - - // When adding a file, we take a lock that gets released after pinning - // is complete, so don't take a second lock here - const lock = Boolean(options.lock) - - if (!lock) { - yield * pinAdd() - return - } - - const release = await gcLock.readLock() - - try { - yield * pinAdd() - } finally { - release() - } - }) +module.exports = ({ addAll }) => { + return async function add (path, options) { // eslint-disable-line require-await + return last(addAll({ + path, + ...options + }, options)) + } } diff --git a/packages/ipfs/src/core/components/pin/pin-manager.js b/packages/ipfs/src/core/components/pin/pin-manager.js index 2b0e894bb0..2755bfd1fe 100644 --- a/packages/ipfs/src/core/components/pin/pin-manager.js +++ b/packages/ipfs/src/core/components/pin/pin-manager.js @@ -10,6 +10,7 @@ const first = require('it-first') const all = require('it-all') const cbor = require('cbor') const multibase = require('multibase') +const multicodec = require('multicodec') // arbitrary limit to the number of concurrent dag operations // const WALK_DAG_CONCURRENCY_LIMIT = 300 @@ -22,7 +23,11 @@ function invalidPinTypeErr (type) { } function toKey (cid) { - return '/' + multibase.encode('base32', cid.multihash).toString().substring(1) + return '/' + multibase.encode('base32upper', cid.multihash).toString().substring(1) +} + +function keyToMultihash (key) { + return multibase.decode(`b${key.toString().slice(1)}`) } const PinTypes = { @@ -60,11 +65,23 @@ class PinManager { async pinDirectly (cid, options = {}) { await this.dag.get(cid, options) - return this.repo.pins.put(toKey(cid), cbor.encode({ - cid: cid.buffer, - type: PinTypes.direct, - metadata: options.metadata - })) + const pin = { + depth: 0 + } + + if (cid.version !== 0) { + pin.version = cid.version + } + + if (cid.codec !== 'dag-pb') { + pin.codec = multicodec.getNumber(cid.codec) + } + + if (options.metadata) { + pin.metadata = options.metadata + } + + return this.repo.pins.put(toKey(cid), cbor.encode(pin)) } async unpin (cid) { // eslint-disable-line require-await @@ -74,11 +91,21 @@ class PinManager { async pinRecursively (cid, options = {}) { await this.fetchCompleteDag(cid, options) - await this.repo.pins.put(toKey(cid), cbor.encode({ - cid: cid.buffer, - type: PinTypes.recursive, - metadata: options.metadata - })) + const pin = {} + + if (cid.version !== 0) { + pin.version = cid.version + } + + if (cid.codec !== 'dag-pb') { + pin.codec = multicodec.getNumber(cid.codec) + } + + if (options.metadata) { + pin.metadata = options.metadata + } + + await this.repo.pins.put(toKey(cid), cbor.encode(pin)) } async * directKeys () { @@ -86,13 +113,16 @@ class PinManager { filters: [(entry) => { const pin = cbor.decode(entry.value) - return pin.type === PinTypes.direct + return pin.depth === 0 }] })) { const pin = cbor.decode(entry.value) + const version = pin.version || 0 + const codec = pin.codec ? multicodec.getName(pin.codec) : 'dag-pb' + const multihash = keyToMultihash(entry.key) yield { - cid: new CID(pin.cid), + cid: new CID(version, codec, multihash), metadata: pin.metadata } } @@ -103,13 +133,16 @@ class PinManager { filters: [(entry) => { const pin = cbor.decode(entry.value) - return pin.type === PinTypes.recursive + return pin.depth === undefined }] })) { const pin = cbor.decode(entry.value) + const version = pin.version || 0 + const codec = pin.codec ? multicodec.getName(pin.codec) : 'dag-pb' + const multihash = keyToMultihash(entry.key) yield { - cid: new CID(pin.cid), + cid: new CID(version, codec, multihash), metadata: pin.metadata } } @@ -154,7 +187,7 @@ class PinManager { const pin = cbor.decode(entry.value) - return types.includes(pin.type) + return types.includes(pin.depth === 0 ? PinTypes.direct : PinTypes.recursive) }], limit: 1 })) @@ -165,7 +198,7 @@ class PinManager { return { cid, pinned: true, - reason: pin.type, + reason: pin.depth === 0 ? PinTypes.direct : PinTypes.recursive, metadata: pin.metadata } } diff --git a/packages/ipfs/src/core/components/pin/rm-all.js b/packages/ipfs/src/core/components/pin/rm-all.js new file mode 100644 index 0000000000..39298fbd7e --- /dev/null +++ b/packages/ipfs/src/core/components/pin/rm-all.js @@ -0,0 +1,46 @@ +'use strict' + +const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const { resolvePath, withTimeoutOption } = require('../../utils') +const { PinTypes } = require('./pin-manager') + +module.exports = ({ pinManager, gcLock, dag }) => { + return withTimeoutOption(async function * rm (paths, options = {}) { + const release = await gcLock.readLock() + + try { + // verify that each hash can be unpinned + for await (const { path, recursive } of normaliseInput(paths)) { + const cid = await resolvePath(dag, path) + const { pinned, reason } = await pinManager.isPinnedWithType(cid, PinTypes.all) + + if (!pinned) { + throw new Error(`${cid} is not pinned`) + } + + switch (reason) { + case (PinTypes.recursive): + if (!recursive) { + throw new Error(`${cid} is pinned recursively`) + } + + await pinManager.unpin(cid) + + yield cid + + break + case (PinTypes.direct): + await pinManager.unpin(cid) + + yield cid + + break + default: + throw new Error(`${cid} is pinned indirectly under ${reason}`) + } + } + } finally { + release() + } + }) +} diff --git a/packages/ipfs/src/core/components/pin/rm.js b/packages/ipfs/src/core/components/pin/rm.js index 39298fbd7e..12a5f3e63d 100644 --- a/packages/ipfs/src/core/components/pin/rm.js +++ b/packages/ipfs/src/core/components/pin/rm.js @@ -1,46 +1,12 @@ 'use strict' -const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') -const { resolvePath, withTimeoutOption } = require('../../utils') -const { PinTypes } = require('./pin-manager') - -module.exports = ({ pinManager, gcLock, dag }) => { - return withTimeoutOption(async function * rm (paths, options = {}) { - const release = await gcLock.readLock() - - try { - // verify that each hash can be unpinned - for await (const { path, recursive } of normaliseInput(paths)) { - const cid = await resolvePath(dag, path) - const { pinned, reason } = await pinManager.isPinnedWithType(cid, PinTypes.all) - - if (!pinned) { - throw new Error(`${cid} is not pinned`) - } - - switch (reason) { - case (PinTypes.recursive): - if (!recursive) { - throw new Error(`${cid} is pinned recursively`) - } - - await pinManager.unpin(cid) - - yield cid - - break - case (PinTypes.direct): - await pinManager.unpin(cid) - - yield cid - - break - default: - throw new Error(`${cid} is pinned indirectly under ${reason}`) - } - } - } finally { - release() - } - }) +const last = require('it-last') + +module.exports = ({ rmAll }) => { + return async function rm (path, options) { // eslint-disable-line require-await + return last(rmAll({ + path, + ...options + }, options)) + } } diff --git a/packages/ipfs/src/core/components/start.js b/packages/ipfs/src/core/components/start.js index 2633afaa0b..b6d5fbbc6a 100644 --- a/packages/ipfs/src/core/components/start.js +++ b/packages/ipfs/src/core/components/start.js @@ -93,10 +93,15 @@ module.exports = ({ tree: Components.dag.tree({ ipld, preload }) } + const pinAddAll = Components.pin.addAll({ pinManager, gcLock, dag }) + const pinRmAll = Components.pin.rmAll({ pinManager, gcLock, dag }) + const pin = { - add: Components.pin.add({ pinManager, gcLock, dag }), + add: Components.pin.add({ addAll: pinAddAll }), + addAll: pinAddAll, ls: Components.pin.ls({ pinManager, dag }), - rm: Components.pin.rm({ pinManager, gcLock, dag }) + rm: Components.pin.rm({ rmAll: pinRmAll }), + rmAll: pinRmAll } // FIXME: resolve this circular dependency diff --git a/packages/ipfs/src/core/components/stop.js b/packages/ipfs/src/core/components/stop.js index 41dea112bb..deefabbf9c 100644 --- a/packages/ipfs/src/core/components/stop.js +++ b/packages/ipfs/src/core/components/stop.js @@ -97,10 +97,15 @@ function createApi ({ stat: Components.object.stat({ ipld, preload }) } + const pinAddAll = Components.pin.addAll({ pinManager, gcLock, dag }) + const pinRmAll = Components.pin.rmAll({ pinManager, gcLock, dag }) + const pin = { - add: Components.pin.add({ pinManager, gcLock, dag }), + add: Components.pin.add({ addAll: pinAddAll }), + addAll: pinAddAll, ls: Components.pin.ls({ pinManager, dag }), - rm: Components.pin.rm({ pinManager, gcLock, dag }) + rm: Components.pin.rm({ rmAll: pinRmAll }), + rmAll: pinRmAll } // FIXME: resolve this circular dependency diff --git a/packages/ipfs/src/core/utils.js b/packages/ipfs/src/core/utils.js index a99990bba2..01ea555fee 100644 --- a/packages/ipfs/src/core/utils.js +++ b/packages/ipfs/src/core/utils.js @@ -81,33 +81,21 @@ const resolvePath = async function (dag, ipfsPath, options) { return new CID(ipfsPath) } - for (const ipfsPath of ipfsPaths) { - if (isIpfs.cid(ipfsPath)) { - cids.push(new CID(ipfsPath)) - continue - } - - const { - cid, - path - } = toCidAndPath(ipfsPath) - - if (!path) { - cids.push(cid) - continue - } + const { + cid, + path + } = toCidAndPath(ipfsPath) - const result = await dag.resolve(cid, { - ...options, - path - }) - - if (CID.isCID(result.cid)) { - cids.push(result.cid) - } + if (!path) { + return cid } - return cid + const result = await dag.resolve(cid, { + ...options, + path + }) + + return result.cid } const mapFile = (file, options) => { diff --git a/packages/ipfs/src/http/api/resources/dag.js b/packages/ipfs/src/http/api/resources/dag.js index be7d6d24c5..63df10932c 100644 --- a/packages/ipfs/src/http/api/resources/dag.js +++ b/packages/ipfs/src/http/api/resources/dag.js @@ -13,7 +13,6 @@ const { const all = require('it-all') const log = debug('ipfs:http-api:dag') log.error = debug('ipfs:http-api:dag:error') -const drain = require('it-drain') const IpldFormats = { get [multicodec.RAW] () { @@ -277,10 +276,6 @@ exports.put = { throw Boom.boomify(err, { message: 'Failed to put node' }) } - if (request.query.pin) { - await drain(ipfs.pin.add(cid)) - } - return h.response({ Cid: { '/': cidToString(cid, { diff --git a/packages/ipfs/src/http/api/resources/pin.js b/packages/ipfs/src/http/api/resources/pin.js index 6503c8b951..8bd02b65ad 100644 --- a/packages/ipfs/src/http/api/resources/pin.js +++ b/packages/ipfs/src/http/api/resources/pin.js @@ -141,7 +141,7 @@ exports.add = { let result try { - result = await all(ipfs.pin.add(cids.map(cid => ({ cid, recursive, metadata })), { + result = await all(ipfs.pin.addAll(cids.map(cid => ({ cid, recursive, metadata })), { signal, timeout })) @@ -206,7 +206,7 @@ exports.rm = { let result try { - result = await all(ipfs.pin.rm(cids.map(cid => ({ cid, recursive })), { + result = await all(ipfs.pin.rmAll(cids.map(cid => ({ cid, recursive })), { signal, timeout })) diff --git a/packages/ipfs/test/cli/pin.js b/packages/ipfs/test/cli/pin.js index 7e74e3fd81..3f8f9efe34 100644 --- a/packages/ipfs/test/cli/pin.js +++ b/packages/ipfs/test/cli/pin.js @@ -24,8 +24,8 @@ describe('pin', () => { before(() => { ipfs = { pin: { - rm: sinon.stub(), - add: sinon.stub(), + rmAll: sinon.stub(), + addAll: sinon.stub(), ls: sinon.stub(), query: sinon.stub() } @@ -42,7 +42,7 @@ describe('pin', () => { } it('recursively (default)', async () => { - ipfs.pin.rm.withArgs([{ + ipfs.pin.rmAll.withArgs([{ ...defaultPinOptions, path: pins.root }], defaultOptions).returns([{ @@ -54,7 +54,7 @@ describe('pin', () => { }) it('non recursively', async () => { - ipfs.pin.rm.withArgs([{ + ipfs.pin.rmAll.withArgs([{ ...defaultPinOptions, path: pins.root, recursive: false @@ -67,7 +67,7 @@ describe('pin', () => { }) it('non recursively (short option)', async () => { - ipfs.pin.rm.withArgs([{ + ipfs.pin.rmAll.withArgs([{ ...defaultPinOptions, path: pins.root, recursive: false @@ -80,7 +80,7 @@ describe('pin', () => { }) it('should rm and print CIDs encoded in specified base', async () => { - ipfs.pin.rm.withArgs([{ + ipfs.pin.rmAll.withArgs([{ ...defaultPinOptions, path: pins.root }], defaultOptions).returns([{ @@ -93,7 +93,7 @@ describe('pin', () => { }) it('with timeout', async () => { - ipfs.pin.rm.withArgs([{ + ipfs.pin.rmAll.withArgs([{ ...defaultPinOptions, path: pins.root }], { @@ -119,7 +119,7 @@ describe('pin', () => { } it('recursively (default)', async () => { - ipfs.pin.add.withArgs([{ + ipfs.pin.addAll.withArgs([{ ...defaultPinOptions, path: pins.root }], defaultOptions).returns([{ @@ -131,7 +131,7 @@ describe('pin', () => { }) it('non recursively', async () => { - ipfs.pin.add.withArgs([{ + ipfs.pin.addAll.withArgs([{ ...defaultPinOptions, path: pins.root, recursive: false @@ -144,7 +144,7 @@ describe('pin', () => { }) it('non recursively (short option)', async () => { - ipfs.pin.add.withArgs([{ + ipfs.pin.addAll.withArgs([{ ...defaultPinOptions, path: pins.root, recursive: false @@ -157,7 +157,7 @@ describe('pin', () => { }) it('with metadata', async () => { - ipfs.pin.add.withArgs([{ + ipfs.pin.addAll.withArgs([{ ...defaultPinOptions, path: pins.root, metadata: { @@ -172,7 +172,7 @@ describe('pin', () => { }) it('with a metadata (short option)', async () => { - ipfs.pin.add.withArgs([{ + ipfs.pin.addAll.withArgs([{ ...defaultPinOptions, path: pins.root, metadata: { @@ -187,7 +187,7 @@ describe('pin', () => { }) it('with json metadata', async () => { - ipfs.pin.add.withArgs([{ + ipfs.pin.addAll.withArgs([{ ...defaultPinOptions, path: pins.root, metadata: { @@ -202,7 +202,7 @@ describe('pin', () => { }) it('should rm and print CIDs encoded in specified base', async () => { - ipfs.pin.add.withArgs([{ + ipfs.pin.addAll.withArgs([{ ...defaultOptions, path: pins.root, recursive: true, @@ -217,7 +217,7 @@ describe('pin', () => { }) it('recursively with timeout', async () => { - ipfs.pin.add.withArgs([{ + ipfs.pin.addAll.withArgs([{ ...defaultPinOptions, path: pins.root }], { diff --git a/packages/ipfs/test/core/gc.spec.js b/packages/ipfs/test/core/gc.spec.js index cef5232229..0313dedbc5 100644 --- a/packages/ipfs/test/core/gc.spec.js +++ b/packages/ipfs/test/core/gc.spec.js @@ -4,7 +4,6 @@ const { expect } = require('interface-ipfs-core/src/utils/mocha') const last = require('it-last') -const drain = require('it-drain') const { Buffer } = require('buffer') const factory = require('../utils/factory') const pEvent = require('p-event') @@ -201,7 +200,7 @@ describe.skip('gc', function () { // Pin first block // Note: pin add will take a read lock const pinLockRequested = pEvent(lockEmitter, 'readLock request') - const pin1 = last(ipfs.pin.add(cid1)) + const pin1 = ipfs.pin.add(cid1) // Once pin lock has been requested, start GC await pinLockRequested @@ -211,7 +210,7 @@ describe.skip('gc', function () { // TODO: Adding pin for removed block never returns, which means the lock // never gets released - // const pin2 = last(ipfs.pin.add(cid2)) + // const pin2 = ipfs.pin.add(cid2) // Confirm second second block has been removed const localRefs = (await ipfs.refs.local()).map(r => r.ref) @@ -232,8 +231,8 @@ describe.skip('gc', function () { const cid2 = (await ipfs.block.put(Buffer.from('block to pin rm 2'), null)).cid // Pin blocks - await drain(ipfs.pin.add(cid1)) - await drain(ipfs.pin.add(cid2)) + await ipfs.pin.add(cid1) + await ipfs.pin.add(cid2) // Unpin first block // Note: pin rm will take a read lock diff --git a/packages/ipfs/test/http-api/inject/pin.js b/packages/ipfs/test/http-api/inject/pin.js index 5d636054fb..de93744863 100644 --- a/packages/ipfs/test/http-api/inject/pin.js +++ b/packages/ipfs/test/http-api/inject/pin.js @@ -20,8 +20,8 @@ describe('/pin', () => { ipfs = { pin: { ls: sinon.stub(), - add: sinon.stub(), - rm: sinon.stub(), + addAll: sinon.stub(), + rmAll: sinon.stub(), query: sinon.stub() } } @@ -29,7 +29,6 @@ describe('/pin', () => { describe('/rm', () => { const defaultOptions = { - recursive: true, signal: sinon.match.instanceOf(AbortSignal), timeout: undefined } @@ -48,9 +47,9 @@ describe('/pin', () => { }) it('unpins recursive pins', async () => { - ipfs.pin.rm.withArgs([cid], defaultOptions).returns([{ + ipfs.pin.rmAll.withArgs([{ cid, recursive: true }], defaultOptions).returns([ cid - }]) + ]) const res = await http({ method: 'POST', @@ -62,12 +61,14 @@ describe('/pin', () => { }) it('unpins direct pins', async () => { - ipfs.pin.rm.withArgs([cid], { - ...defaultOptions, + ipfs.pin.rmAll.withArgs([{ + cid, recursive: false - }).returns([{ + }], { + ...defaultOptions + }).returns([ cid - }]) + ]) const res = await http({ method: 'POST', @@ -79,9 +80,9 @@ describe('/pin', () => { }) it('should remove pin and return base64 encoded CID', async () => { - ipfs.pin.rm.withArgs([cid], defaultOptions).returns([{ + ipfs.pin.rmAll.withArgs([{ cid, recursive: true }], defaultOptions).returns([ cid - }]) + ]) const res = await http({ method: 'POST', @@ -105,12 +106,15 @@ describe('/pin', () => { }) it('accepts a timeout', async () => { - ipfs.pin.rm.withArgs([cid], { + ipfs.pin.rmAll.withArgs([{ + cid, + recursive: true + }], { ...defaultOptions, timeout: 1000 - }).returns([{ + }).returns([ cid - }]) + ]) const res = await http({ method: 'POST', @@ -124,7 +128,6 @@ describe('/pin', () => { describe('/add', () => { const defaultOptions = { - recursive: true, signal: sinon.match.instanceOf(AbortSignal), timeout: undefined } @@ -143,9 +146,13 @@ describe('/pin', () => { }) it('recursively', async () => { - ipfs.pin.add.withArgs([cid], defaultOptions).returns([{ + ipfs.pin.addAll.withArgs([{ + cid, + recursive: true, + metadata: undefined + }], defaultOptions).returns([ cid - }]) + ]) const res = await http({ method: 'POST', @@ -157,12 +164,13 @@ describe('/pin', () => { }) it('directly', async () => { - ipfs.pin.add.withArgs([cid], { - ...defaultOptions, - recursive: false - }).returns([{ + ipfs.pin.addAll.withArgs([{ + cid, + recursive: false, + metadata: undefined + }], defaultOptions).returns([ cid - }]) + ]) const res = await http({ method: 'POST', @@ -174,9 +182,13 @@ describe('/pin', () => { }) it('should add pin and return base64 encoded CID', async () => { - ipfs.pin.add.withArgs([cid], defaultOptions).returns([{ + ipfs.pin.addAll.withArgs([{ + cid, + recursive: true, + metadata: undefined + }], defaultOptions).returns([ cid - }]) + ]) const res = await http({ method: 'POST', @@ -200,12 +212,16 @@ describe('/pin', () => { }) it('accepts a timeout', async () => { - ipfs.pin.add.withArgs([cid], { + ipfs.pin.addAll.withArgs([{ + cid, + recursive: true, + metadata: undefined + }], { ...defaultOptions, timeout: 1000 - }).returns([{ + }).returns([ cid - }]) + ]) const res = await http({ method: 'POST', From b8a2640fb36dd399e5baaf93f1d3781f50f454b4 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 21 Jul 2020 19:16:07 +0100 Subject: [PATCH 14/40] chore: fix tests --- packages/interface-ipfs-core/src/repo/gc.js | 10 +++++----- packages/ipfs/test/core/gc.spec.js | 5 ++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/interface-ipfs-core/src/repo/gc.js b/packages/interface-ipfs-core/src/repo/gc.js index 8cf0b1e7db..bc79697b0b 100644 --- a/packages/interface-ipfs-core/src/repo/gc.js +++ b/packages/interface-ipfs-core/src/repo/gc.js @@ -39,7 +39,7 @@ module.exports = (common, options) => { const pinset = await all(ipfs.pin.ls()) expect(pinset.map(obj => obj.cid.toString())).includes(res.cid.toString()) - await drain(ipfs.pin.rm(res.cid)) + await ipfs.pin.rm(res.cid) await all(ipfs.repo.gc()) const finalPinset = await all(ipfs.pin.ls()) @@ -71,7 +71,7 @@ module.exports = (common, options) => { expect(refsAfterGc.map(r => new CID(r.ref).multihash)).deep.includes(cid.multihash) // Unpin the data - await drain(ipfs.pin.rm(cid)) + await ipfs.pin.rm(cid) // Run garbage collection await all(ipfs.repo.gc()) @@ -159,7 +159,7 @@ module.exports = (common, options) => { expect(refsAfterRmAndGc.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash) // Unpin the data - await drain(ipfs.pin.rm(dataCid)) + await ipfs.pin.rm(dataCid) // Run garbage collection await drain(ipfs.repo.gc()) @@ -179,7 +179,7 @@ module.exports = (common, options) => { const dataCid = addRes.cid // Unpin the data - await drain(ipfs.pin.rm(dataCid)) + await ipfs.pin.rm(dataCid) // Create a link to the data from an object const obj = await new DAGNode(Buffer.from('fruit'), [{ @@ -217,7 +217,7 @@ module.exports = (common, options) => { expect(refsAfterGc.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash) // Recursively unpin the object - await drain(ipfs.pin.rm(objCid.toString())) + await ipfs.pin.rm(objCid.toString()) // Run garbage collection await drain(ipfs.repo.gc()) diff --git a/packages/ipfs/test/core/gc.spec.js b/packages/ipfs/test/core/gc.spec.js index 0313dedbc5..fce8319778 100644 --- a/packages/ipfs/test/core/gc.spec.js +++ b/packages/ipfs/test/core/gc.spec.js @@ -3,7 +3,6 @@ 'use strict' const { expect } = require('interface-ipfs-core/src/utils/mocha') -const last = require('it-last') const { Buffer } = require('buffer') const factory = require('../utils/factory') const pEvent = require('p-event') @@ -237,7 +236,7 @@ describe.skip('gc', function () { // Unpin first block // Note: pin rm will take a read lock const pinLockRequested = pEvent(lockEmitter, 'readLock request') - const pinRm1 = last(ipfs.pin.rm(cid1)) + const pinRm1 = ipfs.pin.rm(cid1) // Once pin lock has been requested, start GC await pinLockRequested @@ -247,7 +246,7 @@ describe.skip('gc', function () { // Once GC has started, start second pin rm await gcStarted - const pinRm2 = last(ipfs.pin.rm(cid2)) + const pinRm2 = ipfs.pin.rm(cid2) const deleted = (await gc).map(i => i.cid.toString()) await pinRm1 From 00e3671d854786a62f63f7f97bb01cff2a80167a Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 21 Jul 2020 19:53:17 +0100 Subject: [PATCH 15/40] chore: fix more tests --- packages/interface-ipfs-core/src/refs-local.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interface-ipfs-core/src/refs-local.js b/packages/interface-ipfs-core/src/refs-local.js index ff8ed0eeb9..f15aa5f6d2 100644 --- a/packages/interface-ipfs-core/src/refs-local.js +++ b/packages/interface-ipfs-core/src/refs-local.js @@ -49,7 +49,7 @@ module.exports = (common, options) => { const imported = await all(importer(dirs, ipfs.block)) // otherwise go-ipfs doesn't show them in the local refs - await ipfs.pin.add(imported.map(i => i.cid)) + await drain(ipfs.pin.addAll(imported.map(i => i.cid))) const refs = await all(ipfs.refs.local()) const cids = refs.map(r => r.ref) From c98b50c580dee11e295c152650c857d0d9ffd1cb Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 21 Jul 2020 20:27:00 +0100 Subject: [PATCH 16/40] chore: make depth infinity for recursive pins --- packages/ipfs/src/core/components/pin/pin-manager.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/ipfs/src/core/components/pin/pin-manager.js b/packages/ipfs/src/core/components/pin/pin-manager.js index 2755bfd1fe..2e339a76b9 100644 --- a/packages/ipfs/src/core/components/pin/pin-manager.js +++ b/packages/ipfs/src/core/components/pin/pin-manager.js @@ -91,7 +91,9 @@ class PinManager { async pinRecursively (cid, options = {}) { await this.fetchCompleteDag(cid, options) - const pin = {} + const pin = { + depth: Infinity + } if (cid.version !== 0) { pin.version = cid.version @@ -133,7 +135,7 @@ class PinManager { filters: [(entry) => { const pin = cbor.decode(entry.value) - return pin.depth === undefined + return pin.depth === Infinity }] })) { const pin = cbor.decode(entry.value) From d4f37962347537dd3e9c0f90a337a353b69410fc Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 21 Jul 2020 21:52:30 +0100 Subject: [PATCH 17/40] chore: keep multihash prefix for key --- .../ipfs/src/core/components/pin/pin-manager.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/ipfs/src/core/components/pin/pin-manager.js b/packages/ipfs/src/core/components/pin/pin-manager.js index 2e339a76b9..7b166c7070 100644 --- a/packages/ipfs/src/core/components/pin/pin-manager.js +++ b/packages/ipfs/src/core/components/pin/pin-manager.js @@ -22,12 +22,12 @@ function invalidPinTypeErr (type) { return errCode(new Error(errMsg), 'ERR_INVALID_PIN_TYPE') } -function toKey (cid) { - return '/' + multibase.encode('base32upper', cid.multihash).toString().substring(1) +function cidToKey (cid) { + return `/${multibase.encoding('base32upper').encode(cid.multihash)}` } function keyToMultihash (key) { - return multibase.decode(`b${key.toString().slice(1)}`) + return multibase.decode(`${key.toString().slice(1)}`) } const PinTypes = { @@ -81,11 +81,11 @@ class PinManager { pin.metadata = options.metadata } - return this.repo.pins.put(toKey(cid), cbor.encode(pin)) + return this.repo.pins.put(cidToKey(cid), cbor.encode(pin)) } async unpin (cid) { // eslint-disable-line require-await - return this.repo.pins.delete(toKey(cid)) + return this.repo.pins.delete(cidToKey(cid)) } async pinRecursively (cid, options = {}) { @@ -107,7 +107,7 @@ class PinManager { pin.metadata = options.metadata } - await this.repo.pins.put(toKey(cid), cbor.encode(pin)) + await this.repo.pins.put(cidToKey(cid), cbor.encode(pin)) } async * directKeys () { @@ -181,7 +181,7 @@ class PinManager { if (recursive || direct || all) { const result = await first(this.repo.pins.query({ - prefix: toKey(cid), + prefix: cidToKey(cid), filters: [entry => { if (all) { return true From 692664dd1263e0295440c74f4aeb950316a2e2b7 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 21 Jul 2020 22:52:22 +0100 Subject: [PATCH 18/40] chore: correct multihash --- packages/ipfs/src/core/components/pin/pin-manager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs/src/core/components/pin/pin-manager.js b/packages/ipfs/src/core/components/pin/pin-manager.js index 7b166c7070..633dd653f2 100644 --- a/packages/ipfs/src/core/components/pin/pin-manager.js +++ b/packages/ipfs/src/core/components/pin/pin-manager.js @@ -23,7 +23,7 @@ function invalidPinTypeErr (type) { } function cidToKey (cid) { - return `/${multibase.encoding('base32upper').encode(cid.multihash)}` + return `/${multibase.encode('base32upper', cid.multihash).toString()}` } function keyToMultihash (key) { From 94204f8c0826447339312f1b6d2eb766b7dad0c4 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 10 Aug 2020 13:44:18 +0100 Subject: [PATCH 19/40] chore: updating deps --- .../browser-add-readable-stream/package.json | 2 +- examples/browser-browserify/package.json | 2 +- .../browser-create-react-app/package.json | 2 +- examples/browser-exchange-files/package.json | 2 +- .../package.json | 2 +- examples/browser-mfs/package.json | 2 +- examples/browser-parceljs/package.json | 2 +- examples/browser-readablestream/package.json | 2 +- examples/browser-script-tag/package.json | 2 +- .../package.json | 2 +- examples/browser-video-streaming/package.json | 2 +- examples/browser-vue/package.json | 2 +- examples/browser-webpack/package.json | 2 +- examples/circuit-relaying/package.json | 2 +- examples/custom-ipfs-repo/package.json | 6 +-- examples/custom-libp2p/package.json | 2 +- .../explore-ethereum-blockchain/package.json | 2 +- .../http-client-browser-pubsub/package.json | 2 +- .../http-client-bundle-webpack/package.json | 2 +- examples/ipfs-101/package.json | 2 +- examples/run-in-electron/package.json | 2 +- examples/running-multiple-nodes/package.json | 2 +- examples/traverse-ipld-graphs/package.json | 10 ++-- packages/interface-ipfs-core/package.json | 18 +++---- packages/ipfs-core-utils/package.json | 4 +- packages/ipfs-http-client/package.json | 18 +++---- .../ipfs-message-port-client/package.json | 8 ++-- .../ipfs-message-port-protocol/package.json | 6 +-- .../ipfs-message-port-server/package.json | 6 +-- packages/ipfs/package.json | 48 +++++++++---------- packages/ipfs/test/cli/general.js | 2 +- 31 files changed, 84 insertions(+), 84 deletions(-) diff --git a/examples/browser-add-readable-stream/package.json b/examples/browser-add-readable-stream/package.json index 3dd00ec40a..ba4fb0fb35 100644 --- a/examples/browser-add-readable-stream/package.json +++ b/examples/browser-add-readable-stream/package.json @@ -12,7 +12,7 @@ "keywords": [], "license": "MIT", "devDependencies": { - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "test-ipfs-example": "^2.0.3" } } diff --git a/examples/browser-browserify/package.json b/examples/browser-browserify/package.json index ac8986a6a2..e4403218e7 100644 --- a/examples/browser-browserify/package.json +++ b/examples/browser-browserify/package.json @@ -18,7 +18,7 @@ "concat-stream": "^2.0.0", "execa": "^4.0.0", "http-server": "^0.11.1", - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "test-ipfs-example": "^2.0.3" }, "browser": { diff --git a/examples/browser-create-react-app/package.json b/examples/browser-create-react-app/package.json index 80a0039df4..ce57755d42 100644 --- a/examples/browser-create-react-app/package.json +++ b/examples/browser-create-react-app/package.json @@ -4,7 +4,7 @@ "private": true, "dependencies": { "dot-prop": "^5.0.0", - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "ipfs-css": "^0.13.1", "react": "^16.8.6", "react-dom": "^16.8.6", diff --git a/examples/browser-exchange-files/package.json b/examples/browser-exchange-files/package.json index aa32c18651..093b4b95c5 100644 --- a/examples/browser-exchange-files/package.json +++ b/examples/browser-exchange-files/package.json @@ -17,7 +17,7 @@ "ipfs-http-client": "^45.0.0" }, "dependencies": { - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "it-all": "^1.0.1", "it-last": "^1.0.1", "test-ipfs-example": "^2.0.3" diff --git a/examples/browser-http-client-upload-file/package.json b/examples/browser-http-client-upload-file/package.json index 8a3f7af350..328c959568 100644 --- a/examples/browser-http-client-upload-file/package.json +++ b/examples/browser-http-client-upload-file/package.json @@ -17,7 +17,7 @@ "ipfs-http-client": "^45.0.0" }, "devDependencies": { - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "parcel-bundler": "^1.12.4", "react": "^16.8.6", "react-dom": "^16.8.6", diff --git a/examples/browser-mfs/package.json b/examples/browser-mfs/package.json index a8a02489a2..97d74e9edf 100644 --- a/examples/browser-mfs/package.json +++ b/examples/browser-mfs/package.json @@ -21,7 +21,7 @@ "webpack-cli": "^3.3.11" }, "dependencies": { - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "mime-sniffer": "~0.0.3" } } diff --git a/examples/browser-parceljs/package.json b/examples/browser-parceljs/package.json index 62ed3f046b..aad3c18112 100644 --- a/examples/browser-parceljs/package.json +++ b/examples/browser-parceljs/package.json @@ -18,7 +18,7 @@ "author": "", "license": "ISC", "dependencies": { - "ipfs": "^0.48.0" + "ipfs": "^0.48.1" }, "devDependencies": { "@babel/cli": "^7.1.5", diff --git a/examples/browser-readablestream/package.json b/examples/browser-readablestream/package.json index 810a1cac31..de871f2cbb 100644 --- a/examples/browser-readablestream/package.json +++ b/examples/browser-readablestream/package.json @@ -20,7 +20,7 @@ "webpack": "^4.43.0" }, "dependencies": { - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "it-to-stream": "^0.1.1", "videostream": "^3.2.0" } diff --git a/examples/browser-script-tag/package.json b/examples/browser-script-tag/package.json index b8cbab2692..e1a11d5f27 100644 --- a/examples/browser-script-tag/package.json +++ b/examples/browser-script-tag/package.json @@ -16,6 +16,6 @@ "test-ipfs-example": "^2.0.3" }, "dependencies": { - "ipfs": "^0.48.0" + "ipfs": "^0.48.1" } } diff --git a/examples/browser-sharing-node-across-tabs/package.json b/examples/browser-sharing-node-across-tabs/package.json index d030c483b5..577a925c82 100644 --- a/examples/browser-sharing-node-across-tabs/package.json +++ b/examples/browser-sharing-node-across-tabs/package.json @@ -23,7 +23,7 @@ "worker-plugin": "4.0.3" }, "dependencies": { - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "ipfs-message-port-client": "^0.0.1", "ipfs-message-port-server": "^0.0.1" }, diff --git a/examples/browser-video-streaming/package.json b/examples/browser-video-streaming/package.json index 1cd932488c..ea4c69ec45 100644 --- a/examples/browser-video-streaming/package.json +++ b/examples/browser-video-streaming/package.json @@ -16,6 +16,6 @@ "test-ipfs-example": "^2.0.3" }, "dependencies": { - "ipfs": "^0.48.0" + "ipfs": "^0.48.1" } } diff --git a/examples/browser-vue/package.json b/examples/browser-vue/package.json index 89aa725912..fc86ec6fa8 100644 --- a/examples/browser-vue/package.json +++ b/examples/browser-vue/package.json @@ -11,7 +11,7 @@ }, "dependencies": { "core-js": "^3.6.4", - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "vue": "^2.6.11" }, "devDependencies": { diff --git a/examples/browser-webpack/package.json b/examples/browser-webpack/package.json index 253b979323..95b2df6e0e 100644 --- a/examples/browser-webpack/package.json +++ b/examples/browser-webpack/package.json @@ -26,7 +26,7 @@ "webpack-dev-server": "^3.11.0" }, "dependencies": { - "ipfs": "^0.48.0" + "ipfs": "^0.48.1" }, "browserslist": [ ">1%", diff --git a/examples/circuit-relaying/package.json b/examples/circuit-relaying/package.json index fa2e44aac0..209669249d 100644 --- a/examples/circuit-relaying/package.json +++ b/examples/circuit-relaying/package.json @@ -15,7 +15,7 @@ "license": "MIT", "dependencies": { "delay": "^4.3.0", - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "ipfs-pubsub-room": "^2.0.1" }, "devDependencies": { diff --git a/examples/custom-ipfs-repo/package.json b/examples/custom-ipfs-repo/package.json index 14a2eabfb9..24c16a271a 100644 --- a/examples/custom-ipfs-repo/package.json +++ b/examples/custom-ipfs-repo/package.json @@ -10,9 +10,9 @@ }, "license": "MIT", "dependencies": { - "datastore-fs": "^1.1.0", - "ipfs": "^0.48.0", - "ipfs-repo": "^5.0.0", + "datastore-fs": "^2.0.0", + "ipfs": "^0.48.1", + "ipfs-repo": "^6.0.1", "it-all": "^1.0.1" }, "devDependencies": { diff --git a/examples/custom-libp2p/package.json b/examples/custom-libp2p/package.json index e4515c27f4..33cba13b77 100644 --- a/examples/custom-libp2p/package.json +++ b/examples/custom-libp2p/package.json @@ -10,7 +10,7 @@ }, "license": "MIT", "dependencies": { - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "libp2p": "^0.28.5", "libp2p-bootstrap": "^0.11.0", "libp2p-kad-dht": "^0.19.9", diff --git a/examples/explore-ethereum-blockchain/package.json b/examples/explore-ethereum-blockchain/package.json index 5140dacb49..33f4055573 100644 --- a/examples/explore-ethereum-blockchain/package.json +++ b/examples/explore-ethereum-blockchain/package.json @@ -10,7 +10,7 @@ "keywords": [], "license": "MIT", "devDependencies": { - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "ipfs-http-client": "^45.0.0", "ipfsd-ctl": "^5.0.0", "test-ipfs-example": "^2.0.3" diff --git a/examples/http-client-browser-pubsub/package.json b/examples/http-client-browser-pubsub/package.json index 553b192ee7..23b19e5343 100644 --- a/examples/http-client-browser-pubsub/package.json +++ b/examples/http-client-browser-pubsub/package.json @@ -20,7 +20,7 @@ "devDependencies": { "execa": "^4.0.0", "go-ipfs": "^0.6.0", - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "ipfsd-ctl": "^5.0.0", "parcel-bundler": "^1.12.4", "test-ipfs-example": "^2.0.3" diff --git a/examples/http-client-bundle-webpack/package.json b/examples/http-client-bundle-webpack/package.json index 23dc379019..f90d6deb19 100644 --- a/examples/http-client-bundle-webpack/package.json +++ b/examples/http-client-bundle-webpack/package.json @@ -23,7 +23,7 @@ "babel-loader": "^8.0.5", "copy-webpack-plugin": "^5.0.4", "execa": "^4.0.0", - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "ipfsd-ctl": "^5.0.0", "react-hot-loader": "^4.12.21", "test-ipfs-example": "^2.0.3", diff --git a/examples/ipfs-101/package.json b/examples/ipfs-101/package.json index dae0e41958..b0b8cc7de7 100644 --- a/examples/ipfs-101/package.json +++ b/examples/ipfs-101/package.json @@ -10,7 +10,7 @@ "author": "David Dias ", "license": "MIT", "dependencies": { - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "it-all": "^1.0.1" }, "devDependencies": { diff --git a/examples/run-in-electron/package.json b/examples/run-in-electron/package.json index 203fc83f7d..1747b6d877 100644 --- a/examples/run-in-electron/package.json +++ b/examples/run-in-electron/package.json @@ -18,7 +18,7 @@ "devDependencies": { "electron": "^6.0.0", "electron-rebuild": "^1.8.4", - "ipfs": "^0.48.0", + "ipfs": "^0.48.1", "test-ipfs-example": "^2.0.3" }, "greenkeeper": { diff --git a/examples/running-multiple-nodes/package.json b/examples/running-multiple-nodes/package.json index b7f88cc0a0..714a16735d 100644 --- a/examples/running-multiple-nodes/package.json +++ b/examples/running-multiple-nodes/package.json @@ -13,6 +13,6 @@ "test-ipfs-example": "^2.0.3" }, "dependencies": { - "ipfs": "^0.48.0" + "ipfs": "^0.48.1" } } diff --git a/examples/traverse-ipld-graphs/package.json b/examples/traverse-ipld-graphs/package.json index a8d8aeb85f..33ff02fec5 100644 --- a/examples/traverse-ipld-graphs/package.json +++ b/examples/traverse-ipld-graphs/package.json @@ -13,10 +13,10 @@ "test-ipfs-example": "^2.0.3" }, "dependencies": { - "cids": "^0.8.3", - "ipfs": "^0.48.0", - "ipld-block": "^0.9.2", - "ipld-dag-pb": "^0.19.0", - "multihashing-async": "^1.0.0" + "cids": "^1.0.0", + "ipfs": "^0.48.1", + "ipld-block": "^0.10.0", + "ipld-dag-pb": "^0.20.0", + "multihashing-async": "^2.0.1" } } diff --git a/packages/interface-ipfs-core/package.json b/packages/interface-ipfs-core/package.json index a225e3d8b7..0d1e593873 100644 --- a/packages/interface-ipfs-core/package.json +++ b/packages/interface-ipfs-core/package.json @@ -36,15 +36,15 @@ "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "chai-subset": "^1.6.0", - "cids": "^0.8.3", + "cids": "^1.0.0", "delay": "^4.3.0", "dirty-chai": "^2.0.1", - "ipfs-unixfs": "^1.0.3", - "ipfs-unixfs-importer": "^2.0.2", + "ipfs-unixfs": "^2.0.2", + "ipfs-unixfs-importer": "^3.0.2", "ipfs-utils": "^2.2.2", - "ipld-block": "^0.9.2", - "ipld-dag-cbor": "^0.16.0", - "ipld-dag-pb": "^0.19.0", + "ipld-block": "^0.10.0", + "ipld-dag-cbor": "^0.17.0", + "ipld-dag-pb": "^0.20.0", "is-ipfs": "^1.0.3", "iso-random-stream": "^1.1.1", "it-all": "^1.0.1", @@ -54,15 +54,15 @@ "it-map": "^1.0.2", "it-pushable": "^1.3.1", "multiaddr": "^7.4.3", - "multibase": "^1.0.1", - "multihashing-async": "^1.0.0", + "multibase": "^3.0.0", + "multihashing-async": "^2.0.1", "nanoid": "^3.0.2", "peer-id": "^0.13.12", "readable-stream": "^3.4.0", "temp-write": "^4.0.0" }, "devDependencies": { - "aegir": "^23.0.0", + "aegir": "^25.0.0", "ipfsd-ctl": "^5.0.0" }, "contributors": [ diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index b08de20343..300ed0560f 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -31,7 +31,7 @@ "blob-to-it": "0.0.1", "browser-readablestream-to-it": "0.0.1", "buffer": "^5.6.0", - "cids": "^0.8.3", + "cids": "^1.0.0", "err-code": "^2.0.0", "ipfs-utils": "^2.2.2", "it-all": "^1.0.1", @@ -39,7 +39,7 @@ "it-peekable": "0.0.1" }, "devDependencies": { - "aegir": "^23.0.0", + "aegir": "^25.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "delay": "^4.3.0", diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 5e9b7f974d..d7e922951b 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -44,15 +44,15 @@ "any-signal": "^1.1.0", "bignumber.js": "^9.0.0", "buffer": "^5.6.0", - "cids": "^0.8.3", + "cids": "^1.0.0", "debug": "^4.1.0", "form-data": "^3.0.0", "ipfs-core-utils": "^0.3.0", "ipfs-utils": "^2.2.2", - "ipld-block": "^0.9.2", - "ipld-dag-cbor": "^0.16.0", - "ipld-dag-pb": "^0.19.0", - "ipld-raw": "^5.0.0", + "ipld-block": "^0.10.0", + "ipld-dag-cbor": "^0.17.0", + "ipld-dag-pb": "^0.20.0", + "ipld-raw": "^6.0.0", "iso-url": "^0.4.7", "it-last": "^1.0.1", "it-tar": "^1.2.2", @@ -61,16 +61,16 @@ "merge-options": "^2.0.0", "multiaddr": "^7.4.3", "multiaddr-to-uri": "^5.1.0", - "multibase": "^1.0.1", - "multicodec": "^1.0.0", - "multihashes": "^1.0.1", + "multibase": "^3.0.0", + "multicodec": "^2.0.0", + "multihashes": "^3.0.1", "nanoid": "^3.0.2", "node-fetch": "^2.6.0", "parse-duration": "^0.4.4", "stream-to-it": "^0.2.1" }, "devDependencies": { - "aegir": "^23.0.0", + "aegir": "^25.0.0", "cross-env": "^7.0.0", "go-ipfs": "^0.6.0", "interface-ipfs-core": "^0.138.0", diff --git a/packages/ipfs-message-port-client/package.json b/packages/ipfs-message-port-client/package.json index 350a97ffec..c8d08384fb 100644 --- a/packages/ipfs-message-port-client/package.json +++ b/packages/ipfs-message-port-client/package.json @@ -34,15 +34,15 @@ "dep-check": "aegir dep-check" }, "dependencies": { - "cids": "^0.8.3", + "cids": "^1.0.0", "browser-readablestream-to-it": "0.0.1" }, "devDependencies": { "ipfs-message-port-protocol": "~0.0.1", "ipfs-message-port-server": "~0.0.1", - "ipld-dag-pb": "^0.19.0", - "ipfs": "^0.48.0", - "aegir": "^23.0.0", + "ipld-dag-pb": "^0.20.0", + "ipfs": "^0.48.1", + "aegir": "^25.0.0", "cross-env": "^7.0.0", "interface-ipfs-core": "^0.138.0" }, diff --git a/packages/ipfs-message-port-protocol/package.json b/packages/ipfs-message-port-protocol/package.json index 68520d9ff7..4af1723ce8 100644 --- a/packages/ipfs-message-port-protocol/package.json +++ b/packages/ipfs-message-port-protocol/package.json @@ -33,11 +33,11 @@ }, "dependencies": { "buffer": "^5.6.0", - "cids": "^0.8.3", - "ipld-block": "^0.9.2" + "cids": "^1.0.0", + "ipld-block": "^0.10.0" }, "devDependencies": { - "aegir": "^23.0.0", + "aegir": "^25.0.0", "interface-ipfs-core": "^0.138.0" }, "engines": { diff --git a/packages/ipfs-message-port-server/package.json b/packages/ipfs-message-port-server/package.json index 7ffe733aa9..ed325de943 100644 --- a/packages/ipfs-message-port-server/package.json +++ b/packages/ipfs-message-port-server/package.json @@ -34,14 +34,14 @@ "dep-check": "aegir dep-check" }, "dependencies": { - "cids": "^0.8.3", + "cids": "^1.0.0", "it-all": "^1.0.1" }, "devDependencies": { "ipfs-message-port-protocol": "~0.0.1", "@types/it-all": "^1.0.0", - "ipfs": "^0.48.0", - "aegir": "^23.0.0", + "ipfs": "^0.48.1", + "aegir": "^25.0.0", "cross-env": "^7.0.0", "interface-ipfs-core": "^0.138.0" }, diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index f7dbbd302f..6d2cade4aa 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -77,11 +77,11 @@ "byteman": "^1.3.5", "cbor": "^5.0.1", "cid-tool": "^0.4.0", - "cids": "^0.8.3", + "cids": "^1.0.0", "class-is": "^1.1.0", "dag-cbor-links": "^1.3.3", - "datastore-core": "^1.1.0", - "datastore-pubsub": "^0.3.2", + "datastore-core": "^2.0.0", + "datastore-pubsub": "^0.4.0", "debug": "^4.1.0", "dlv": "^1.1.3", "err-code": "^2.0.0", @@ -92,26 +92,26 @@ "hamt-sharding": "^1.0.0", "hapi-pino": "^6.1.0", "hashlru": "^2.3.0", - "interface-datastore": "^1.0.2", + "interface-datastore": "^2.0.0", "ipfs-bitswap": "^2.0.1", - "ipfs-block-service": "^0.17.1", + "ipfs-block-service": "^0.18.0", "ipfs-core-utils": "^0.3.0", "ipfs-http-client": "^45.0.0", "ipfs-http-response": "^0.5.0", - "ipfs-repo": "^5.0.0", - "ipfs-unixfs": "^1.0.3", - "ipfs-unixfs-exporter": "^2.0.2", - "ipfs-unixfs-importer": "^2.0.2", + "ipfs-repo": "^6.0.1", + "ipfs-unixfs": "^2.0.2", + "ipfs-unixfs-exporter": "^3.0.2", + "ipfs-unixfs-importer": "^3.0.2", "ipfs-utils": "^2.2.2", - "ipld": "^0.26.2", - "ipld-bitcoin": "^0.3.0", - "ipld-block": "^0.9.2", - "ipld-dag-cbor": "^0.16.0", - "ipld-dag-pb": "^0.19.0", - "ipld-ethereum": "^4.0.0", - "ipld-git": "^0.5.0", - "ipld-raw": "^5.0.0", - "ipld-zcash": "^0.4.0", + "ipld": "^0.27.1", + "ipld-bitcoin": "^0.4.0", + "ipld-block": "^0.10.0", + "ipld-dag-cbor": "^0.17.0", + "ipld-dag-pb": "^0.20.0", + "ipld-ethereum": "^5.0.1", + "ipld-git": "^0.6.1", + "ipld-raw": "^6.0.0", + "ipld-zcash": "^0.5.0", "ipns": "^0.7.1", "is-domain-name": "^1.0.1", "is-ipfs": "^1.0.3", @@ -141,7 +141,7 @@ "libp2p-mdns": "^0.14.1", "libp2p-mplex": "^0.9.5", "libp2p-noise": "^1.1.1", - "libp2p-record": "^0.7.3", + "libp2p-record": "^0.8.0", "libp2p-secio": "^0.12.6", "libp2p-tcp": "^0.14.5", "libp2p-webrtc-star": "^0.18.0", @@ -151,16 +151,16 @@ "mortice": "^2.0.0", "multiaddr": "^7.4.3", "multiaddr-to-uri": "^5.1.0", - "multibase": "^1.0.1", - "multicodec": "^1.0.0", - "multihashing-async": "^1.0.0", + "multibase": "^3.0.0", + "multicodec": "^2.0.0", + "multihashing-async": "^2.0.1", "p-defer": "^3.0.0", "p-queue": "^6.1.0", "parse-duration": "^0.4.4", "peer-id": "^0.13.12", "pretty-bytes": "^5.3.0", "progress": "^2.0.1", - "protons": "^1.2.1", + "protons": "^2.0.0", "semver": "^7.3.2", "stream-to-it": "^0.2.1", "streaming-iterables": "^5.0.0", @@ -173,7 +173,7 @@ "yargs-promise": "^1.1.0" }, "devDependencies": { - "aegir": "^23.0.0", + "aegir": "^25.0.0", "base64url": "^3.0.1", "clear-module": "^4.0.0", "cross-env": "^7.0.0", diff --git a/packages/ipfs/test/cli/general.js b/packages/ipfs/test/cli/general.js index 8945cbddaf..5feae2d923 100644 --- a/packages/ipfs/test/cli/general.js +++ b/packages/ipfs/test/cli/general.js @@ -30,7 +30,7 @@ describe('general cli options', () => { }) }) -describe('--migrate', () => { +describe.only('--migrate', () => { let ipfs, repoPath async function setRepoVersion (version) { From ea82cf92c8201c6966314ab00e1c13c5a3add538 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 11 Aug 2020 14:19:25 +0100 Subject: [PATCH 20/40] chore: update deps --- packages/ipfs-http-client/package.json | 2 +- packages/ipfs/package.json | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index d7e922951b..281497c113 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -59,7 +59,7 @@ "it-to-buffer": "^1.0.0", "it-to-stream": "^0.1.1", "merge-options": "^2.0.0", - "multiaddr": "^7.4.3", + "multiaddr": "^8.0.0", "multiaddr-to-uri": "^5.1.0", "multibase": "^3.0.0", "multicodec": "^2.0.0", diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 6d2cade4aa..db70e17b2a 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -114,7 +114,7 @@ "ipld-zcash": "^0.5.0", "ipns": "^0.7.1", "is-domain-name": "^1.0.1", - "is-ipfs": "^1.0.3", + "is-ipfs": "^2.0.0", "iso-url": "^0.4.7", "it-all": "^1.0.1", "it-concat": "^1.0.0", @@ -131,25 +131,25 @@ "jsondiffpatch": "^0.4.1", "just-safe-set": "^2.1.0", "libp2p": "^0.28.5", - "libp2p-bootstrap": "^0.11.0", - "libp2p-crypto": "^0.17.8", + "libp2p-bootstrap": "^0.12.0", + "libp2p-crypto": "^0.18.0", "libp2p-delegated-content-routing": "^0.5.0", "libp2p-delegated-peer-routing": "^0.5.0", - "libp2p-floodsub": "^0.21.0", + "libp2p-floodsub": "^0.22.0", "libp2p-gossipsub": "^0.4.0", - "libp2p-kad-dht": "^0.19.9", + "libp2p-kad-dht": "^0.20.0", "libp2p-mdns": "^0.14.1", "libp2p-mplex": "^0.9.5", "libp2p-noise": "^1.1.1", - "libp2p-record": "^0.8.0", - "libp2p-secio": "^0.12.6", - "libp2p-tcp": "^0.14.5", + "libp2p-record": "^0.9.0", + "libp2p-secio": "^0.13.0", + "libp2p-tcp": "^0.15.0", "libp2p-webrtc-star": "^0.18.0", "libp2p-websockets": "^0.13.3", - "mafmt": "^7.0.0", + "mafmt": "^8.0.0", "merge-options": "^2.0.0", "mortice": "^2.0.0", - "multiaddr": "^7.4.3", + "multiaddr": "^8.0.0", "multiaddr-to-uri": "^5.1.0", "multibase": "^3.0.0", "multicodec": "^2.0.0", @@ -157,7 +157,7 @@ "p-defer": "^3.0.0", "p-queue": "^6.1.0", "parse-duration": "^0.4.4", - "peer-id": "^0.13.12", + "peer-id": "^0.14.0", "pretty-bytes": "^5.3.0", "progress": "^2.0.1", "protons": "^2.0.0", From e8c2543de6bb959c7677c27af91f96fd4428a51f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 14 Aug 2020 19:22:23 +0100 Subject: [PATCH 21/40] chore: replace node buffers with uint8arrays --- docs/BROWSERS.md | 2 +- docs/MIGRATION-TO-ASYNC-AWAIT.md | 89 ++++++++++------- docs/core-api/BLOCK.md | 9 +- docs/core-api/FILES.md | 20 ++-- docs/core-api/OBJECT.md | 6 +- docs/core-api/PUBSUB.md | 2 +- examples/browser-exchange-files/public/app.js | 7 +- examples/browser-exchange-files/test.js | 2 +- examples/browser-parceljs/public/index.js | 3 +- .../browser-webpack/src/components/app.js | 6 +- examples/custom-ipfs-repo/custom-lock.js | 3 +- examples/custom-ipfs-repo/index.js | 6 +- examples/custom-libp2p/package.json | 14 +-- examples/custom-libp2p/test.js | 3 +- examples/http-client-browser-pubsub/index.js | 5 +- examples/http-client-name-api/index.js | 4 +- examples/ipfs-101/1.js | 5 +- examples/ipfs-101/README.md | 2 +- examples/test-ipfs-example/utils.js | 5 +- examples/traverse-ipld-graphs/eth.js | 5 +- .../get-path-accross-formats.js | 3 +- examples/traverse-ipld-graphs/git.js | 5 +- examples/traverse-ipld-graphs/tree.js | 3 +- packages/interface-ipfs-core/package.json | 19 ++-- packages/interface-ipfs-core/src/add-all.js | 20 ++-- packages/interface-ipfs-core/src/add.js | 24 ++--- packages/interface-ipfs-core/src/block/get.js | 17 ++-- packages/interface-ipfs-core/src/block/put.js | 18 ++-- packages/interface-ipfs-core/src/block/rm.js | 24 ++--- .../interface-ipfs-core/src/block/stat.js | 4 +- packages/interface-ipfs-core/src/cat.js | 53 +++++----- .../interface-ipfs-core/src/config/set.js | 4 +- packages/interface-ipfs-core/src/dag/get.js | 18 ++-- packages/interface-ipfs-core/src/dag/put.js | 6 +- .../interface-ipfs-core/src/dag/resolve.js | 8 +- packages/interface-ipfs-core/src/dag/tree.js | 4 +- packages/interface-ipfs-core/src/dht/get.js | 3 +- .../interface-ipfs-core/src/dht/provide.js | 12 +-- packages/interface-ipfs-core/src/dht/utils.js | 4 +- .../interface-ipfs-core/src/files/chmod.js | 10 +- packages/interface-ipfs-core/src/files/cp.js | 24 ++--- packages/interface-ipfs-core/src/files/ls.js | 10 +- packages/interface-ipfs-core/src/files/mv.js | 19 ++-- .../interface-ipfs-core/src/files/read.js | 52 +++++----- .../interface-ipfs-core/src/files/stat.js | 14 +-- .../interface-ipfs-core/src/files/touch.js | 17 ++-- .../interface-ipfs-core/src/files/write.js | 98 +++++++++---------- packages/interface-ipfs-core/src/get.js | 29 +++--- packages/interface-ipfs-core/src/ls.js | 19 ++-- .../src/miscellaneous/resolve.js | 6 +- .../src/name-pubsub/cancel.js | 6 +- .../interface-ipfs-core/src/name/publish.js | 4 +- .../interface-ipfs-core/src/name/resolve.js | 18 ++-- .../interface-ipfs-core/src/object/data.js | 6 +- .../interface-ipfs-core/src/object/get.js | 24 ++--- .../interface-ipfs-core/src/object/links.js | 18 ++-- .../src/object/patch/add-link.js | 8 +- .../src/object/patch/append-data.js | 8 +- .../src/object/patch/rm-link.js | 6 +- .../src/object/patch/set-data.js | 8 +- .../interface-ipfs-core/src/object/put.js | 20 ++-- .../interface-ipfs-core/src/object/stat.js | 14 +-- packages/interface-ipfs-core/src/pin/add.js | 5 +- .../interface-ipfs-core/src/pubsub/publish.js | 6 +- .../src/pubsub/subscribe.js | 53 +++++----- .../interface-ipfs-core/src/refs-local.js | 5 +- packages/interface-ipfs-core/src/refs.js | 3 +- packages/interface-ipfs-core/src/repo/gc.js | 14 +-- .../src/utils/create-sharded-directory.js | 3 +- .../src/utils/create-two-shards.js | 3 +- .../src/utils/traverse-leaf-nodes.js | 3 +- packages/ipfs-core-utils/package.json | 10 +- .../normalise-input/normalise-content.js | 4 +- .../ipfs-core-utils/src/to-cid-and-path.js | 2 +- .../test/files/format-mode.spec.js | 2 +- .../test/files/format-mtime.spec.js | 2 +- .../test/files/normalise-input.spec.js | 7 +- .../test/pins/normalise-input.spec.js | 7 +- packages/ipfs-core-utils/test/utils/chai.js | 10 -- packages/ipfs-http-client/package.json | 12 +-- packages/ipfs-http-client/src/block/get.js | 3 +- .../ipfs-http-client/src/config/replace.js | 4 +- packages/ipfs-http-client/src/dag/get.js | 8 +- .../ipfs-http-client/src/dht/find-peer.js | 3 +- packages/ipfs-http-client/src/dht/get.js | 7 +- packages/ipfs-http-client/src/files/read.js | 5 +- packages/ipfs-http-client/src/get.js | 6 +- packages/ipfs-http-client/src/index.js | 4 +- packages/ipfs-http-client/src/ls.js | 3 +- packages/ipfs-http-client/src/object/data.js | 5 +- packages/ipfs-http-client/src/object/get.js | 6 +- packages/ipfs-http-client/src/object/links.js | 3 +- .../src/object/patch/add-link.js | 3 +- .../src/object/patch/append-data.js | 3 +- .../src/object/patch/rm-link.js | 3 +- .../src/object/patch/set-data.js | 3 +- packages/ipfs-http-client/src/object/put.js | 15 +-- packages/ipfs-http-client/src/object/stat.js | 3 +- .../ipfs-http-client/src/pubsub/subscribe.js | 12 +-- packages/ipfs-http-client/src/refs/index.js | 3 +- .../ipfs-http-client/test/commands.spec.js | 2 +- .../ipfs-http-client/test/constructor.spec.js | 2 +- packages/ipfs-http-client/test/dag.spec.js | 16 +-- packages/ipfs-http-client/test/diag.spec.js | 2 +- .../test/endpoint-config.spec.js | 2 +- .../ipfs-http-client/test/exports.spec.js | 4 +- packages/ipfs-http-client/test/files.spec.js | 6 +- packages/ipfs-http-client/test/get.spec.js | 5 +- packages/ipfs-http-client/test/key.spec.js | 2 +- .../test/lib.error-handler.spec.js | 2 +- packages/ipfs-http-client/test/log.spec.js | 6 +- .../test/node/custom-headers.js | 7 +- .../ipfs-http-client/test/node/request-api.js | 6 +- packages/ipfs-http-client/test/node/swarm.js | 2 +- packages/ipfs-http-client/test/ping.spec.js | 2 +- packages/ipfs-http-client/test/repo.spec.js | 2 +- packages/ipfs-http-client/test/stats.spec.js | 2 +- .../ipfs-http-client/test/sub-modules.spec.js | 2 +- .../ipfs-message-port-client/package.json | 4 +- packages/ipfs-message-port-protocol/README.md | 7 +- .../ipfs-message-port-protocol/package.json | 3 +- .../ipfs-message-port-protocol/src/cid.js | 1 - .../ipfs-message-port-protocol/src/rpc.ts | 27 +++-- .../test/block.browser.js | 10 +- .../test/cid.browser.js | 2 +- .../test/cid.spec.js | 2 +- .../test/core.browser.js | 68 ++++++------- .../test/dag.browser.js | 10 +- .../test/dag.spec.js | 8 +- .../ipfs-message-port-server/package.json | 2 +- .../ipfs-message-port-server/src/block.js | 8 +- packages/ipfs-message-port-server/src/ipfs.ts | 5 +- .../test/basic.spec.js | 2 +- packages/ipfs/.aegir.js | 9 ++ packages/ipfs/package.json | 45 +++++---- packages/ipfs/src/cli/commands/block/get.js | 3 +- packages/ipfs/src/cli/commands/cat.js | 3 +- packages/ipfs/src/cli/commands/dag/get.js | 8 +- packages/ipfs/src/cli/commands/dag/put.js | 5 +- packages/ipfs/src/cli/commands/object/get.js | 6 +- packages/ipfs/src/cli/commands/pubsub/pub.js | 4 +- packages/ipfs/src/cli/commands/refs-local.js | 4 +- packages/ipfs/src/core/components/dht.js | 17 ++-- .../core/components/files/utils/hamt-utils.js | 3 +- .../files/utils/to-async-iterator.js | 6 +- .../components/files/utils/update-mfs-root.js | 2 +- .../components/files/utils/with-mfs-root.js | 2 +- .../ipfs/src/core/components/files/write.js | 3 +- packages/ipfs/src/core/components/id.js | 3 +- packages/ipfs/src/core/components/init.js | 7 +- .../ipfs/src/core/components/object/get.js | 8 +- .../ipfs/src/core/components/object/new.js | 3 +- .../components/object/patch/append-data.js | 4 +- .../ipfs/src/core/components/object/put.js | 9 +- .../src/core/components/pin/pin-manager.js | 6 +- packages/ipfs/src/core/index.js | 1 - packages/ipfs/src/core/ipns/publisher.js | 17 ++-- packages/ipfs/src/core/ipns/republisher.js | 3 +- packages/ipfs/src/core/ipns/resolver.js | 14 +-- .../core/ipns/routing/offline-datastore.js | 23 +++-- .../src/core/ipns/routing/pubsub-datastore.js | 17 ++-- packages/ipfs/src/core/ipns/routing/utils.js | 8 +- packages/ipfs/src/core/utils.js | 3 +- packages/ipfs/src/http/api/resources/block.js | 1 - .../ipfs/src/http/api/resources/config.js | 1 - packages/ipfs/src/http/api/resources/dag.js | 18 ++-- .../ipfs/src/http/api/resources/object.js | 1 - .../src/http/gateway/resources/gateway.js | 5 +- .../http/utils/multipart-request-parser.js | 7 +- packages/ipfs/test/cli/add.js | 8 +- packages/ipfs/test/cli/bitswap.js | 2 +- packages/ipfs/test/cli/block.js | 8 +- packages/ipfs/test/cli/bootstrap.js | 2 +- packages/ipfs/test/cli/cat.js | 20 ++-- packages/ipfs/test/cli/commands.js | 2 +- packages/ipfs/test/cli/config.js | 2 +- packages/ipfs/test/cli/daemon.js | 2 +- packages/ipfs/test/cli/dag.js | 27 ++--- packages/ipfs/test/cli/dht.js | 2 +- packages/ipfs/test/cli/dns.js | 2 +- packages/ipfs/test/cli/files/chmod.js | 2 +- packages/ipfs/test/cli/files/cp.js | 2 +- packages/ipfs/test/cli/files/flush.js | 2 +- packages/ipfs/test/cli/files/ls.js | 2 +- packages/ipfs/test/cli/files/mkdir.js | 2 +- packages/ipfs/test/cli/files/mv.js | 2 +- packages/ipfs/test/cli/files/read.js | 2 +- packages/ipfs/test/cli/files/rm.js | 2 +- packages/ipfs/test/cli/files/stat.js | 2 +- packages/ipfs/test/cli/files/touch.js | 2 +- packages/ipfs/test/cli/files/write.js | 2 +- packages/ipfs/test/cli/general.js | 7 +- packages/ipfs/test/cli/get.js | 5 +- packages/ipfs/test/cli/id.js | 2 +- packages/ipfs/test/cli/init.js | 6 +- packages/ipfs/test/cli/key.js | 2 +- packages/ipfs/test/cli/ls.js | 2 +- packages/ipfs/test/cli/name-pubsub.js | 2 +- packages/ipfs/test/cli/name.js | 2 +- packages/ipfs/test/cli/object.js | 14 +-- packages/ipfs/test/cli/pin.js | 2 +- packages/ipfs/test/cli/ping.js | 2 +- packages/ipfs/test/cli/progress-bar.js | 2 +- packages/ipfs/test/cli/pubsub.js | 8 +- packages/ipfs/test/cli/refs-local.js | 6 +- packages/ipfs/test/cli/refs.js | 2 +- packages/ipfs/test/cli/repo.js | 2 +- packages/ipfs/test/cli/swarm.js | 2 +- packages/ipfs/test/cli/version.js | 2 +- packages/ipfs/test/core/add-all.spec.js | 2 +- packages/ipfs/test/core/bitswap.spec.js | 8 +- packages/ipfs/test/core/block.spec.js | 8 +- packages/ipfs/test/core/bootstrap.spec.js | 2 +- packages/ipfs/test/core/circuit-relay.spec.js | 2 +- packages/ipfs/test/core/create-node.spec.js | 2 +- packages/ipfs/test/core/dag.spec.js | 6 +- packages/ipfs/test/core/dht.spec.js | 10 +- packages/ipfs/test/core/exports.spec.js | 4 +- .../ipfs/test/core/files-sharding.spec.js | 6 +- packages/ipfs/test/core/files.spec.js | 8 +- packages/ipfs/test/core/gc.spec.js | 28 +++--- packages/ipfs/test/core/id.spec.js | 2 +- packages/ipfs/test/core/init.spec.js | 6 +- packages/ipfs/test/core/ipld.spec.js | 4 +- packages/ipfs/test/core/kad-dht.node.js | 8 +- packages/ipfs/test/core/key-exchange.spec.js | 2 +- packages/ipfs/test/core/libp2p.spec.js | 2 +- packages/ipfs/test/core/mfs-preload.spec.js | 6 +- packages/ipfs/test/core/name-pubsub.js | 19 ++-- packages/ipfs/test/core/name.spec.js | 2 +- packages/ipfs/test/core/object.spec.js | 24 ++--- packages/ipfs/test/core/ping.spec.js | 2 +- packages/ipfs/test/core/preload.spec.js | 66 ++++++------- packages/ipfs/test/core/pubsub.spec.js | 6 +- packages/ipfs/test/core/stats.spec.js | 2 +- packages/ipfs/test/core/utils.js | 2 +- packages/ipfs/test/gateway/index.js | 10 +- packages/ipfs/test/http-api/inject/bitswap.js | 2 +- packages/ipfs/test/http-api/inject/block.js | 3 +- .../ipfs/test/http-api/inject/bootstrap.js | 2 +- .../test/http-api/inject/browser-headers.js | 2 +- packages/ipfs/test/http-api/inject/config.js | 3 +- packages/ipfs/test/http-api/inject/dag.js | 15 ++- packages/ipfs/test/http-api/inject/dht.js | 3 +- packages/ipfs/test/http-api/inject/dns.js | 2 +- packages/ipfs/test/http-api/inject/files.js | 3 +- packages/ipfs/test/http-api/inject/id.js | 2 +- packages/ipfs/test/http-api/inject/key.js | 2 +- .../ipfs/test/http-api/inject/mfs/chmod.js | 2 +- packages/ipfs/test/http-api/inject/mfs/cp.js | 2 +- .../ipfs/test/http-api/inject/mfs/flush.js | 2 +- packages/ipfs/test/http-api/inject/mfs/ls.js | 2 +- .../ipfs/test/http-api/inject/mfs/mkdir.js | 2 +- packages/ipfs/test/http-api/inject/mfs/mv.js | 2 +- .../ipfs/test/http-api/inject/mfs/read.js | 6 +- packages/ipfs/test/http-api/inject/mfs/rm.js | 2 +- .../ipfs/test/http-api/inject/mfs/stat.js | 2 +- .../ipfs/test/http-api/inject/mfs/touch.js | 2 +- .../ipfs/test/http-api/inject/mfs/write.js | 3 +- packages/ipfs/test/http-api/inject/name.js | 2 +- packages/ipfs/test/http-api/inject/object.js | 7 +- packages/ipfs/test/http-api/inject/pin.js | 2 +- packages/ipfs/test/http-api/inject/ping.js | 2 +- packages/ipfs/test/http-api/inject/pubsub.js | 3 +- packages/ipfs/test/http-api/inject/repo.js | 2 +- packages/ipfs/test/http-api/inject/resolve.js | 2 +- packages/ipfs/test/http-api/inject/stats.js | 2 +- packages/ipfs/test/http-api/inject/swarm.js | 2 +- packages/ipfs/test/http-api/inject/version.js | 2 +- packages/ipfs/test/utils/test-http-method.js | 2 +- 270 files changed, 1078 insertions(+), 1082 deletions(-) delete mode 100644 packages/ipfs-core-utils/test/utils/chai.js diff --git a/docs/BROWSERS.md b/docs/BROWSERS.md index 9f68f61e63..b5f4b7f972 100644 --- a/docs/BROWSERS.md +++ b/docs/BROWSERS.md @@ -66,7 +66,7 @@ document.addEventListener('DOMContentLoaded', async () => { const cid = results[0].hash console.log('CID created via ipfs.add:', cid) const data = await node.cat(cid) - console.log('Data read back via ipfs.cat:', data.toString()) + console.log('Data read back via ipfs.cat:', new TextDecoder().decode(data)) }) ``` diff --git a/docs/MIGRATION-TO-ASYNC-AWAIT.md b/docs/MIGRATION-TO-ASYNC-AWAIT.md index 73f3fcf83b..2760ba9c31 100644 --- a/docs/MIGRATION-TO-ASYNC-AWAIT.md +++ b/docs/MIGRATION-TO-ASYNC-AWAIT.md @@ -171,9 +171,10 @@ e.g. ```js const readable = ipfs.catReadableStream('QmHash') +const decoder = new TextDecoder() readable.on('data', chunk => { - console.log(chunk.toString()) + console.log(decoder.decode(chunk)) }) readable.on('end', () => { @@ -185,9 +186,10 @@ Becomes: ```js const source = ipfs.cat('QmHash') +const decoder = new TextDecoder() for await (const chunk of source) { - console.log(chunk.toString()) + console.log(decoder.decode(chunk)) } console.log('done') @@ -201,9 +203,10 @@ e.g. ```js const readable = ipfs.catReadableStream('QmHash') +const decoder = new TextDecoder() readable.on('data', chunk => { - console.log(chunk.toString()) + console.log(decoder.decode(chunk)) }) readable.on('end', () => { @@ -216,9 +219,10 @@ Becomes: ```js const toStream = require('it-to-stream') const readable = toStream.readable(ipfs.cat('QmHash')) +const decoder = new TextDecoder() readable.on('data', chunk => { - console.log(chunk.toString()) + console.log(decoder.decode(chunk)) }) readable.on('end', () => { @@ -238,11 +242,12 @@ e.g. ```js const { pipeline, Writable } = require('stream') +const decoder = new TextDecoder() -let data = Buffer.alloc(0) +let data = new Uint8Array(0) const concat = new Writable({ write (chunk, enc, cb) { - data = Buffer.concat([data, chunk]) + data = uint8ArrayConcat([data, chunk]) cb() } }) @@ -251,7 +256,7 @@ pipeline( ipfs.catReadableStream('QmHash'), concat, err => { - console.log(data.toString()) + console.log(decoder.decode(chunk)) } ) ``` @@ -260,11 +265,12 @@ Becomes: ```js const pipe = require('it-pipe') +const decoder = new TextDecoder() -let data = Buffer.alloc(0) +let data = new Uint8Array(0) const concat = async source => { for await (const chunk of source) { - data = Buffer.concat([data, chunk]) + data = uint8ArrayConcat([data, chunk]) } } @@ -273,15 +279,16 @@ const data = await pipe( concat ) -console.log(data.toString()) +console.log(decoder.decode(data)) ``` ...which, by the way, could more succinctly be written as: ```js const toBuffer = require('it-to-buffer') +const decoder = new TextDecoder() const data = await toBuffer(ipfs.cat('QmHash')) -console.log(data.toString()) +console.log(decoder.decode(data)) ``` **Impact 🍏** @@ -292,11 +299,12 @@ e.g. ```js const { pipeline, Writable } = require('stream') +const decoder = new TextDecoder() -let data = Buffer.alloc(0) +let data = new Uint8Array(0) const concat = new Writable({ write (chunk, enc, cb) { - data = Buffer.concat([data, chunk]) + data = uint8ArrayConcat([data, chunk]) cb() } }) @@ -305,7 +313,7 @@ pipeline( ipfs.catReadableStream('QmHash'), concat, err => { - console.log(data.toString()) + console.log(decoder.decode(data)) } ) ``` @@ -315,11 +323,12 @@ Becomes: ```js const toStream = require('it-to-stream') const { pipeline, Writable } = require('stream') +const decoder = new TextDecoder() -let data = Buffer.alloc(0) +let data = new Uint8Array(0) const concat = new Writable({ write (chunk, enc, cb) { - data = Buffer.concat([data, chunk]) + data = uint8ArrayConcat([data, chunk]) cb() } }) @@ -328,7 +337,7 @@ pipeline( toStream.readable(ipfs.cat('QmHash')), concat, err => { - console.log(data.toString()) + console.log(decoder.decode(data)) } ) ``` @@ -472,10 +481,12 @@ Use a [for/await](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Refere e.g. ```js +const decoder = new TextDecoder() + pull( ipfs.catPullStream('QmHash'), pull.through(chunk => { - console.log(chunk.toString()) + console.log(decoder.decode(data)) }), pull.onEnd(err => { console.log('done') @@ -486,8 +497,10 @@ pull( Becomes: ```js +const decoder = new TextDecoder() + for await (const chunk of ipfs.cat('QmHash')) { - console.log(chunk.toString()) + console.log(decoder.decode(data)) } console.log('done') @@ -500,10 +513,12 @@ Convert the async iterable to a pull stream. e.g. ```js +const decoder = new TextDecoder() + pull( ipfs.catPullStream('QmHash'), pull.through(chunk => { - console.log(chunk.toString()) + console.log(decoder.decode(data)) }), pull.onEnd(err => { console.log('done') @@ -515,11 +530,12 @@ Becomes: ```js const toPull = require('async-iterator-to-pull-stream') +const decoder = new TextDecoder() pull( toPull.source(ipfs.cat('QmHash')), pull.through(chunk => { - console.log(chunk.toString()) + console.log(decoder.decode(data)) }), pull.onEnd(err => { console.log('done') @@ -538,10 +554,12 @@ Use `it-pipe` and `it-concat` concat data from an async iterable. e.g. ```js +const decoder = new TextDecoder() + pull( ipfs.catPullStream('QmHash'), pull.collect((err, chunks) => { - console.log(Buffer.concat(chunks).toString()) + console.log(decoder.decode(uint8ArrayConcat(chunks))) }) ) ``` @@ -551,13 +569,14 @@ Becomes: ```js const pipe = require('it-pipe') const concat = require('it-concat') +const decoder = new TextDecoder() const data = await pipe( ipfs.cat('QmHash'), concat ) -console.log(data.toString()) +console.log(decoder.decode(data)) ``` #### Transform Pull Streams @@ -640,8 +659,8 @@ e.g. ```js const results = await ipfs.addAll([ - { path: 'root/1.txt', content: Buffer.from('one') }, - { path: 'root/2.txt', content: Buffer.from('two') } + { path: 'root/1.txt', content: 'one' }, + { path: 'root/2.txt', content: 'two' } ]) // Note that ALL files have already been added to IPFS @@ -654,8 +673,8 @@ Becomes: ```js const addSource = ipfs.addAll([ - { path: 'root/1.txt', content: Buffer.from('one') }, - { path: 'root/2.txt', content: Buffer.from('two') } + { path: 'root/1.txt', content: 'one' }, + { path: 'root/2.txt', content: 'two' } ]) for await (const file of addSource) { @@ -669,8 +688,8 @@ Alternatively you can buffer up the results using the `it-all` utility: const all = require('it-all') const results = await all(ipfs.addAll([ - { path: 'root/1.txt', content: Buffer.from('one') }, - { path: 'root/2.txt', content: Buffer.from('two') } + { path: 'root/1.txt', content: 'one' }, + { path: 'root/2.txt', content: 'two' } ])) results.forEach(file => { @@ -682,8 +701,8 @@ Often you just want the last item (the root directory entry) when adding multipl ```js const results = await ipfs.addAll([ - { path: 'root/1.txt', content: Buffer.from('one') }, - { path: 'root/2.txt', content: Buffer.from('two') } + { path: 'root/1.txt', content: 'one' }, + { path: 'root/2.txt', content: 'two' } ]) const lastResult = results[results.length - 1] @@ -695,8 +714,8 @@ Becomes: ```js const addSource = ipfs.addAll([ - { path: 'root/1.txt', content: Buffer.from('one') }, - { path: 'root/2.txt', content: Buffer.from('two') } + { path: 'root/1.txt', content: 'one' }, + { path: 'root/2.txt', content: 'two' } ]) let lastResult @@ -711,8 +730,8 @@ Alternatively you can use the `it-last` utility: ```js const lastResult = await last(ipfs.addAll([ - { path: 'root/1.txt', content: Buffer.from('one') }, - { path: 'root/2.txt', content: Buffer.from('two') } + { path: 'root/1.txt', content: 'one' }, + { path: 'root/2.txt', content: 'two' } ])) console.log(lastResult) diff --git a/docs/core-api/BLOCK.md b/docs/core-api/BLOCK.md index e7562e04eb..aa80b6821f 100644 --- a/docs/core-api/BLOCK.md +++ b/docs/core-api/BLOCK.md @@ -92,11 +92,12 @@ An optional object which may have the following keys: ```JavaScript // Defaults -const buf = Buffer.from('a serialized object') +const buf = new TextEncoder().encode('a serialized object') +const decoder = new TextDecoder() const block = await ipfs.block.put(buf) -console.log(block.data.toString()) +console.log(decoder.decode(block.data)) // Logs: // a serialized object console.log(block.cid.toString()) @@ -105,12 +106,12 @@ console.log(block.cid.toString()) // With custom format and hashtype through CID const CID = require('cids') -const buf = Buffer.from('another serialized object') +const buf = new TextEncoder().encode('another serialized object') const cid = new CID(1, 'dag-pb', multihash) const block = await ipfs.block.put(blob, cid) -console.log(block.data.toString()) +console.log(decoder.decode(block.data)) // Logs: // a serialized object console.log(block.cid.toString()) diff --git a/docs/core-api/FILES.md b/docs/core-api/FILES.md index 21827f1a7b..b2611a023e 100644 --- a/docs/core-api/FILES.md +++ b/docs/core-api/FILES.md @@ -420,11 +420,9 @@ An optional object which may have the following keys: #### Example ```JavaScript -const chunks = [] for await (const chunk of ipfs.cat(ipfsPath)) { - chunks.push(chunk) + console.info(chunk) } -console.log(Buffer.concat(chunks).toString()) ``` A great source of [examples](https://github.com/ipfs/js-ipfs/blob/master/packages/interface-ipfs-core/src/cat.js) can be found in the tests for this API. @@ -459,7 +457,7 @@ Each yielded object is of the form: ```js { path: string, - content: >, + content: >, mode: number, mtime: { secs: number, nsecs: number } } @@ -470,7 +468,6 @@ Here, each `path` corresponds to the name of a file, and `content` is an async i #### Example ```JavaScript -const BufferList = require('bl/BufferList') const cid = 'QmQ2r6iMNpky5f1m4cnm3Yqw8VSvjuKpTcK1X7dBR1LkJF' for await (const file of ipfs.get(cid)) { @@ -478,12 +475,13 @@ for await (const file of ipfs.get(cid)) { if (!file.content) continue; - const content = new BufferList() - for await (const chunk of file.content) { - content.append(chunk) + const content = [] + +for await (const chunk of file.content) { + content.push(chunk) } - console.log(content.toString()) + console.log(content) } ``` @@ -855,7 +853,7 @@ for await (const chunk of ipfs.files.read('/hello-world')) { chunks.push(chunk) } -console.log(Buffer.concat(chunks).toString()) +console.log(uint8ArrayConcat(chunks).toString()) // Hello, World! ``` @@ -899,7 +897,7 @@ An optional object which may have the following keys: #### Example ```JavaScript -await ipfs.files.write('/hello-world', Buffer.from('Hello, world!')) +await ipfs.files.write('/hello-world', new TextEncoder().encode('Hello, world!')) ``` ### `ipfs.files.mv(...from, to, [options])` diff --git a/docs/core-api/OBJECT.md b/docs/core-api/OBJECT.md index 2e231627a7..faff11c532 100644 --- a/docs/core-api/OBJECT.md +++ b/docs/core-api/OBJECT.md @@ -122,7 +122,7 @@ An optional object which may have the following keys: ```JavaScript const obj = { - Data: Buffer.from('Some data'), + Data: new TextEncoder().encode('Some data'), Links: [] } @@ -451,7 +451,7 @@ An optional object which may have the following keys: ### Example ```JavaScript -const cid = await ipfs.object.patch.appendData(multihash, Buffer.from('more data')) +const cid = await ipfs.object.patch.appendData(multihash, new TextEncoder().encode('more data')) ``` A great source of [examples][] can be found in the tests for this API. @@ -486,7 +486,7 @@ An optional object which may have the following keys: ```JavaScript const cid = '/ipfs/Qmfoo' -const updatedCid = await ipfs.object.patch.setData(cid, Buffer.from('more data')) +const updatedCid = await ipfs.object.patch.setData(cid, new TextEncoder().encode('more data')) ``` A great source of [examples][] can be found in the tests for this API. diff --git a/docs/core-api/PUBSUB.md b/docs/core-api/PUBSUB.md index e66118514f..cd4a62bf36 100644 --- a/docs/core-api/PUBSUB.md +++ b/docs/core-api/PUBSUB.md @@ -137,7 +137,7 @@ If **only** the `topic` param is provided, unsubscribe will remove **all** handl ```JavaScript const topic = 'fruit-of-the-day' -const msg = Buffer.from('banana') +const msg = new TextEncoder().encode('banana') await ipfs.pubsub.publish(topic, msg) diff --git a/examples/browser-exchange-files/public/app.js b/examples/browser-exchange-files/public/app.js index 3a7f05d6c0..a00792f54a 100644 --- a/examples/browser-exchange-files/public/app.js +++ b/examples/browser-exchange-files/public/app.js @@ -3,7 +3,8 @@ const IPFS = require('ipfs') const all = require('it-all') -const { Buffer } = IPFS +const uint8ArrayConcat = require('uint8arrays/concat') +const uint8ArrayFromString = require('uint8arrays/from-string') // Node const $nodeId = document.querySelector('.node-id') @@ -157,7 +158,7 @@ const workspaceUpdated = async () => { } const publishHash = (hash) => { - const data = Buffer.from(hash) + const data = uint8ArrayFromString(hash) return node.pubsub.publish(workspace, data) } @@ -223,7 +224,7 @@ async function getFile () { for await (const file of node.get(hash)) { if (file.content) { - const content = Buffer.concat(await all(file.content)) + const content = uint8ArrayConcat(await all(file.content)) await appendFile(file.name, hash, file.size, content) onSuccess(`The ${file.name} file was added.`) diff --git a/examples/browser-exchange-files/test.js b/examples/browser-exchange-files/test.js index c190c1efae..208b3448fe 100644 --- a/examples/browser-exchange-files/test.js +++ b/examples/browser-exchange-files/test.js @@ -194,7 +194,7 @@ module.exports[pkg.name] = function (browser) { .then(res => res.text()) .then(done, done) }, [ url ], ({ value: contents }) => { - browser.expect(contents.toString('utf8')).to.equal(FILE_CONTENT) + browser.expect(uint8ArrayToString(contents, 'utf8')).to.equal(FILE_CONTENT) }) }) } diff --git a/examples/browser-parceljs/public/index.js b/examples/browser-parceljs/public/index.js index c0d6f4b6fa..7010fb7848 100644 --- a/examples/browser-parceljs/public/index.js +++ b/examples/browser-parceljs/public/index.js @@ -1,5 +1,6 @@ import 'babel-polyfill' import IPFS from 'ipfs' +import uint8ArrayConcat from 'uint8arrays/concat' document.addEventListener('DOMContentLoaded', async () => { // IPFS node setup @@ -34,5 +35,5 @@ document.addEventListener('DOMContentLoaded', async () => { buffers.push(buf) } - log(`The contents of the file was: ${Buffer.concat(buffers).toString()}`) + log(`The contents of the file was: ${uint8ArrayConcat(buffers).toString()}`) }) diff --git a/examples/browser-webpack/src/components/app.js b/examples/browser-webpack/src/components/app.js index a06a2e398d..f6b7ad4e91 100644 --- a/examples/browser-webpack/src/components/app.js +++ b/examples/browser-webpack/src/components/app.js @@ -2,6 +2,8 @@ const React = require('react') const IPFS = require('ipfs') +const uint8ArrayConcat = require('uint8arrays/concat') +const uint8ArrayToString = require('uint8arrays/to-string') const stringToUse = 'hello world from webpacked IPFS' @@ -39,8 +41,8 @@ class App extends React.Component { bufs.push(buf) } - const data = Buffer.concat(bufs) - this.setState({ addedFileContents: data.toString('utf8') }) + const data = uint8ArrayConcat(bufs) + this.setState({ addedFileContents: uint8ArrayToString(data) }) } render () { diff --git a/examples/custom-ipfs-repo/custom-lock.js b/examples/custom-ipfs-repo/custom-lock.js index 45c625ff57..a8b36dbdac 100644 --- a/examples/custom-ipfs-repo/custom-lock.js +++ b/examples/custom-ipfs-repo/custom-lock.js @@ -1,6 +1,7 @@ 'use strict' const PATH = require('path') +const uint8ArrayFromString = require('uint8arrays/from-string') /** * Uses an object in an S3 bucket as a lock to signal that an IPFS repo is in use. @@ -39,7 +40,7 @@ class S3Lock { } // There's no lock yet, create one - this.s3.put(lockPath, Buffer.from(''), (err, data) => { + this.s3.put(lockPath, uint8ArrayFromString(''), (err, data) => { if (err) { return callback(err, null) } diff --git a/examples/custom-ipfs-repo/index.js b/examples/custom-ipfs-repo/index.js index 6640aa773d..d73c0086a0 100644 --- a/examples/custom-ipfs-repo/index.js +++ b/examples/custom-ipfs-repo/index.js @@ -4,6 +4,8 @@ const IPFS = require('ipfs') const Repo = require('ipfs-repo') const fsLock = require('ipfs-repo/src/lock') const all = require('it-all') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayConcat = require('uint8arrays/concat') // Create our custom options const customRepositoryOptions = { @@ -82,12 +84,12 @@ async function main () { // Once we have the version, let's add a file to IPFS const file = await node.add({ path: 'test-data.txt', - content: Buffer.from('We are using a customized repo!') + content: uint8ArrayFromString('We are using a customized repo!') }) // Log out the added files metadata and cat the file from IPFS console.log('\nAdded file:', file.path, file.cid) - const data = Buffer.concat(await all(node.cat(file.cid))) + const data = uint8ArrayConcat(await all(node.cat(file.cid))) // Print out the files contents to console console.log('\nFetched file content:') diff --git a/examples/custom-libp2p/package.json b/examples/custom-libp2p/package.json index 4fec22348c..65e10ce997 100644 --- a/examples/custom-libp2p/package.json +++ b/examples/custom-libp2p/package.json @@ -11,13 +11,13 @@ "license": "MIT", "dependencies": { "ipfs": "^0.49.0", - "libp2p": "^0.28.5", - "libp2p-bootstrap": "^0.11.0", - "libp2p-kad-dht": "^0.19.9", - "libp2p-mdns": "^0.14.1", - "libp2p-mplex": "^0.9.5", - "libp2p-secio": "^0.12.6", - "libp2p-tcp": "^0.14.5" + "libp2p": "libp2p/js-libp2p#fix/replace-node-buffers-with-uint8arrays", + "libp2p-bootstrap": "^0.12.0", + "libp2p-kad-dht": "^0.20.0", + "libp2p-mdns": "^0.15.0", + "libp2p-mplex": "^0.10.0", + "libp2p-secio": "^0.13.0", + "libp2p-tcp": "^0.15.0" }, "devDependencies": { "execa": "^4.0.0", diff --git a/examples/custom-libp2p/test.js b/examples/custom-libp2p/test.js index d4d80ffc6a..0e527f69bd 100644 --- a/examples/custom-libp2p/test.js +++ b/examples/custom-libp2p/test.js @@ -7,6 +7,7 @@ const TCP = require('libp2p-tcp') const MPLEX = require('libp2p-mplex') const SECIO = require('libp2p-secio') const PeerId = require('peer-id') +const uint8ArrayToString = require('uint8arrays/to-string') async function test () { let output = '' @@ -18,7 +19,7 @@ async function test () { proc.all.on('data', async (data) => { process.stdout.write(data) - output += data.toString('utf8') + output += uint8ArrayToString(data) if (output.includes('The node now has')) { // the node has started up, try to dial it diff --git a/examples/http-client-browser-pubsub/index.js b/examples/http-client-browser-pubsub/index.js index 1e9c7421b2..0270734b41 100644 --- a/examples/http-client-browser-pubsub/index.js +++ b/examples/http-client-browser-pubsub/index.js @@ -2,6 +2,7 @@ const IpfsHttpClient = require('ipfs-http-client') const { sleep, Logger, onEnterPress, catchAndLog } = require('./util') +const uint8ArrayToString = require('uint8arrays/to-string') async function main () { const apiUrlInput = document.getElementById('api-url') @@ -76,13 +77,13 @@ async function main () { await ipfs.pubsub.subscribe(nextTopic, msg => { const from = msg.from - const seqno = msg.seqno.toString('hex') + const seqno = uint8ArrayToString(msg.seqno, 'base16') if (from === peerId) return log(`Ignoring message ${seqno} from self`) log(`Message ${seqno} from ${from}:`) try { log(JSON.stringify(msg.data.toString(), null, 2)) } catch (_) { - log(msg.data.toString('hex')) + log(uint8ArrayToString(msg.data, 'base16')) } }) diff --git a/examples/http-client-name-api/index.js b/examples/http-client-name-api/index.js index 1742976219..1a725c76da 100644 --- a/examples/http-client-name-api/index.js +++ b/examples/http-client-name-api/index.js @@ -1,6 +1,6 @@ /* eslint-disable no-console */ 'use strict' -const { Buffer } = require('buffer') + const ipfsHttp = require('ipfs-http-client') let ipfs @@ -61,7 +61,7 @@ const addAndPublish = async (e) => { e.preventDefault() const input = e.target.elements.text - const buffer = Buffer.from(input.value) + const buffer = new TextEncoder().encode(input.value) showStatus('Adding to IPFS...', COLORS.active) try { diff --git a/examples/ipfs-101/1.js b/examples/ipfs-101/1.js index 67cc05e79b..a277e7096b 100644 --- a/examples/ipfs-101/1.js +++ b/examples/ipfs-101/1.js @@ -2,6 +2,7 @@ const IPFS = require('ipfs') const all = require('it-all') +const uint8ArrayConcat = require('uint8arrays/concat') async function main () { const node = await IPFS.create() @@ -11,12 +12,12 @@ async function main () { const file = await node.add({ path: 'hello.txt', - content: Buffer.from('Hello World 101') + content: uint8ArrayFromString('Hello World 101') }) console.log('Added file:', file.path, file.cid.toString()) - const data = Buffer.concat(await all(node.cat(file.cid))) + const data = uint8ArrayConcat(await all(node.cat(file.cid))) console.log('Added file contents:', data.toString()) } diff --git a/examples/ipfs-101/README.md b/examples/ipfs-101/README.md index 2db1113ef3..ab622c7353 100644 --- a/examples/ipfs-101/README.md +++ b/examples/ipfs-101/README.md @@ -116,7 +116,7 @@ async function main () { chunks.push(chunk) } - console.log('Added file contents:', Buffer.concat(chunks).toString()) + console.log('Added file contents:', uint8ArrayConcat(chunks).toString()) } main() diff --git a/examples/test-ipfs-example/utils.js b/examples/test-ipfs-example/utils.js index 6261efd2c1..a4f32a00f1 100644 --- a/examples/test-ipfs-example/utils.js +++ b/examples/test-ipfs-example/utils.js @@ -4,6 +4,7 @@ const fs = require('fs-extra') const path = require('path') const execa = require('execa') const which = require('which') +const uint8ArrayToString = require('uint8arrays/to-string') async function startServer (dir) { async function serveFrom (appDir) { @@ -17,7 +18,7 @@ async function startServer (dir) { proc.all.on('data', (data) => { process.stdout.write(data) - const line = data.toString('utf8') + const line = uint8ArrayToString(data) output += line if (output.includes('Hit CTRL-C to stop the server')) { @@ -127,7 +128,7 @@ async function waitForOutput (expectedOutput, command, args = [], opts = {}) { proc.all.on('data', (data) => { process.stdout.write(data) - output += data.toString('utf8') + output += uint8ArrayToString(data) if (output.includes(expectedOutput)) { foundExpectedOutput = true diff --git a/examples/traverse-ipld-graphs/eth.js b/examples/traverse-ipld-graphs/eth.js index 5cd32765bf..ab7154a561 100644 --- a/examples/traverse-ipld-graphs/eth.js +++ b/examples/traverse-ipld-graphs/eth.js @@ -6,6 +6,7 @@ const multihashing = require('multihashing-async') const Block = require('ipld-block') const CID = require('cids') const fs = require('fs').promises +const uint8ArrayToString = require('uint8arrays/to-string') async function main () { const ipfs = await createNode() @@ -32,10 +33,10 @@ async function main () { let res res = await ipfs.dag.get(block302516 + '/number') - console.log(res.value.toString('hex')) + console.log(uint8ArrayToString(res.value, 'base16')) res = await ipfs.dag.get(block302517 + '/parent/number') - console.log(res.value.toString('hex')) + console.log(uint8ArrayToString(res.value, 'base16')) } main() diff --git a/examples/traverse-ipld-graphs/get-path-accross-formats.js b/examples/traverse-ipld-graphs/get-path-accross-formats.js index 051845e6a8..387ee40473 100644 --- a/examples/traverse-ipld-graphs/get-path-accross-formats.js +++ b/examples/traverse-ipld-graphs/get-path-accross-formats.js @@ -4,13 +4,14 @@ const createNode = require('./create-node') const { DAGNode } = require('ipld-dag-pb') +const uint8ArrayFromString = require('uint8arrays/from-string') async function main () { const ipfs = await createNode() console.log('\nStart of the example:') - const someData = Buffer.from('capoeira') + const someData = uint8ArrayFromString('capoeira') const pbNode = new DAGNode(someData) const pbNodeCid = await ipfs.dag.put(pbNode, { diff --git a/examples/traverse-ipld-graphs/git.js b/examples/traverse-ipld-graphs/git.js index 6c6dcf1751..78dece69c4 100644 --- a/examples/traverse-ipld-graphs/git.js +++ b/examples/traverse-ipld-graphs/git.js @@ -6,6 +6,7 @@ const multihashing = require('multihashing-async') const Block = require('ipld-block') const CID = require('cids') const fs = require('fs').promises +const uint8ArrayToString = require('uint8arrays/to-string') async function main () { const ipfs = await createNode() @@ -40,8 +41,8 @@ async function main () { async function logResult (fn, comment) { const result = await fn() - if (Buffer.isBuffer(result.value)) { // Blobs (files) are returned as buffer instance - result.value = result.value.toString() + if (result.value instanceof Uint8Array) { // Blobs (files) are returned as buffer instance + result.value = uint8ArrayToString(result.value) } console.log('-'.repeat(80)) diff --git a/examples/traverse-ipld-graphs/tree.js b/examples/traverse-ipld-graphs/tree.js index 1acba5e58d..d9754591c1 100644 --- a/examples/traverse-ipld-graphs/tree.js +++ b/examples/traverse-ipld-graphs/tree.js @@ -4,13 +4,14 @@ const createNode = require('./create-node') const { DAGNode } = require('ipld-dag-pb') +const uint8ArrayFromString = require('uint8arrays/from-string') async function main () { const ipfs = await createNode() console.log('\nStart of the example:') - const someData = Buffer.from('capoeira') + const someData = uint8ArrayFromString('capoeira') const pbNode = new DAGNode(someData) const pbNodeCid = await ipfs.dag.put(pbNode, { diff --git a/packages/interface-ipfs-core/package.json b/packages/interface-ipfs-core/package.json index 2c6d96c0fd..ef6874ab80 100644 --- a/packages/interface-ipfs-core/package.json +++ b/packages/interface-ipfs-core/package.json @@ -32,7 +32,6 @@ "license": "MIT", "dependencies": { "abort-controller": "^3.0.0", - "buffer": "^5.6.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "chai-subset": "^1.6.0", @@ -41,30 +40,30 @@ "dirty-chai": "^2.0.1", "ipfs-unixfs": "^2.0.2", "ipfs-unixfs-importer": "^3.0.2", - "ipfs-utils": "^2.2.2", + "ipfs-utils": "ipfs/js-ipfs-utils#revert-54-xhr", "ipld-block": "^0.10.0", "ipld-dag-cbor": "^0.17.0", "ipld-dag-pb": "^0.20.0", - "is-ipfs": "^1.0.3", + "is-ipfs": "^2.0.0", "iso-random-stream": "^1.1.1", "it-all": "^1.0.1", - "it-concat": "^1.0.0", "it-drain": "^1.0.1", "it-last": "^1.0.1", "it-map": "^1.0.2", "it-pushable": "^1.3.1", - "libp2p-crypto": "^0.17.9", - "multiaddr": "^7.4.3", + "libp2p-crypto": "^0.18.0", + "multiaddr": "^8.0.0", "multibase": "^3.0.0", "multihashing-async": "^2.0.1", "nanoid": "^3.0.2", - "peer-id": "^0.13.12", + "peer-id": "^0.14.0", "readable-stream": "^3.4.0", - "temp-write": "^4.0.0" + "temp-write": "^4.0.0", + "uint8arrays": "^1.1.0" }, "devDependencies": { - "aegir": "^25.0.0", - "ipfsd-ctl": "^5.0.0" + "aegir": "^26.0.0", + "ipfsd-ctl": "^6.0.0" }, "contributors": [ "Alan Shaw ", diff --git a/packages/interface-ipfs-core/src/add-all.js b/packages/interface-ipfs-core/src/add-all.js index 1ad872d7e7..21c0f889f7 100644 --- a/packages/interface-ipfs-core/src/add-all.js +++ b/packages/interface-ipfs-core/src/add-all.js @@ -1,7 +1,6 @@ /* eslint-env mocha, browser */ 'use strict' -const { Buffer } = require('buffer') const { fixtures } = require('./utils') const { Readable } = require('readable-stream') const all = require('it-all') @@ -15,6 +14,7 @@ const globSource = require('ipfs-utils/src/files/glob-source') const { isNode } = require('ipfs-utils/src/env') const { getDescribe, getIt, expect } = require('./utils/mocha') const testTimeout = require('./utils/test-timeout') +const uint8ArrayFromString = require('uint8arrays/from-string') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -33,7 +33,7 @@ module.exports = (common, options) => { async function testMode (mode, expectedMode) { const content = String(Math.random() + Date.now()) const files = await all(ipfs.addAll([{ - content: Buffer.from(content), + content: uint8ArrayFromString(content), mode }])) expect(files).to.have.length(1) @@ -46,7 +46,7 @@ module.exports = (common, options) => { async function testMtime (mtime, expectedMtime) { const content = String(Math.random() + Date.now()) const files = await all(ipfs.addAll({ - content: Buffer.from(content), + content: uint8ArrayFromString(content), mtime })) expect(files).to.have.length(1) @@ -61,7 +61,7 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when adding files', () => { - return testTimeout(() => drain(ipfs.addAll(Buffer.from('Hello'), { + return testTimeout(() => drain(ipfs.addAll(uint8ArrayFromString('Hello'), { timeout: 1 }))) }) @@ -78,7 +78,7 @@ module.exports = (common, options) => { expect(filesAdded[0].cid.toString()).to.be.eq('QmTVfLxf3qXiJgr4KwG6UBckcNvTqBp93Rwy5f7h3mHsVC') }) - it('should add a Buffer as array of tuples', async () => { + it('should add a Uint8Array as array of tuples', async () => { const tuple = { path: 'testfile.txt', content: fixtures.smallFile.data } const filesAdded = await all(ipfs.addAll([tuple])) @@ -94,7 +94,7 @@ module.exports = (common, options) => { const expectedCid = 'QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS' const rs = new Readable() - rs.push(Buffer.from('some data')) + rs.push(uint8ArrayFromString('some data')) rs.push(null) const tuple = { path: 'data.txt', content: rs } @@ -217,10 +217,10 @@ module.exports = (common, options) => { const files = await all(ipfs.addAll([{ path: '/foo/bar.txt', - content: Buffer.from(content) + content: uint8ArrayFromString(content) }, { path: '/foo/baz.txt', - content: Buffer.from(content) + content: uint8ArrayFromString(content) }], { onlyHash: true })) expect(files).to.have.length(3) @@ -348,7 +348,7 @@ module.exports = (common, options) => { }) it('should respect raw leaves when file is smaller than one block and no metadata is present', async () => { - const files = await all(ipfs.addAll(Buffer.from([0, 1, 2]), { + const files = await all(ipfs.addAll(Uint8Array.from([0, 1, 2]), { cidVersion: 1, rawLeaves: true })) @@ -361,7 +361,7 @@ module.exports = (common, options) => { it('should override raw leaves when file is smaller than one block and metadata is present', async () => { const files = await all(ipfs.addAll({ - content: Buffer.from([0, 1, 2]), + content: Uint8Array.from([0, 1, 2]), mode: 0o123, mtime: { secs: 1000, diff --git a/packages/interface-ipfs-core/src/add.js b/packages/interface-ipfs-core/src/add.js index 25a5a683d9..e34c867f63 100644 --- a/packages/interface-ipfs-core/src/add.js +++ b/packages/interface-ipfs-core/src/add.js @@ -1,7 +1,6 @@ /* eslint-env mocha, browser */ 'use strict' -const { Buffer } = require('buffer') const { fixtures } = require('./utils') const { Readable } = require('readable-stream') const { supportsFileReader } = require('ipfs-utils/src/supports') @@ -11,6 +10,7 @@ const { getDescribe, getIt, expect } = require('./utils/mocha') const testTimeout = require('./utils/test-timeout') const echoUrl = (text) => `${process.env.ECHO_SERVER}/download?data=${encodeURIComponent(text)}` const redirectUrl = (url) => `${process.env.ECHO_SERVER}/redirect?to=${encodeURI(url)}` +const uint8ArrayFromString = require('uint8arrays/from-string') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -29,7 +29,7 @@ module.exports = (common, options) => { async function testMode (mode, expectedMode) { const content = String(Math.random() + Date.now()) const file = await ipfs.add({ - content: Buffer.from(content), + content, mode }) expect(file).to.have.property('mode', expectedMode) @@ -41,7 +41,7 @@ module.exports = (common, options) => { async function testMtime (mtime, expectedMtime) { const content = String(Math.random() + Date.now()) const file = await ipfs.add({ - content: Buffer.from(content), + content, mtime }) expect(file).to.have.deep.property('mtime', expectedMtime) @@ -55,7 +55,7 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when adding files', () => { - return testTimeout(() => ipfs.add(Buffer.from('Hello'), { + return testTimeout(() => ipfs.add('Hello', { timeout: 1 })) }) @@ -149,7 +149,7 @@ module.exports = (common, options) => { const expectedCid = 'QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS' const rs = new Readable() - rs.push(Buffer.from('some data')) + rs.push(uint8ArrayFromString('some data')) rs.push(null) const file = await ipfs.add(rs) @@ -180,7 +180,7 @@ module.exports = (common, options) => { this.slow(10 * 1000) const content = String(Math.random() + Date.now()) - const file = await ipfs.add(Buffer.from(content), { onlyHash: true }) + const file = await ipfs.add(content, { onlyHash: true }) await expect(ipfs.object.get(file.cid, { timeout: 4000 })) .to.eventually.be.rejected() @@ -243,7 +243,7 @@ module.exports = (common, options) => { const [result, expectedResult] = await Promise.all([ ipfs.add(urlSource(url)), - ipfs.add(Buffer.from(text)) + ipfs.add(text) ]) expect(result.err).to.not.exist() @@ -258,7 +258,7 @@ module.exports = (common, options) => { const [result, expectedResult] = await Promise.all([ ipfs.add(urlSource(redirectUrl(url))), - ipfs.add(Buffer.from(text)) + ipfs.add(text) ]) expect(result.err).to.not.exist() @@ -285,7 +285,7 @@ module.exports = (common, options) => { const [result, expectedResult] = await Promise.all([ ipfs.add(urlSource(url), addOpts), - ipfs.add({ path: 'download', content: Buffer.from(filename) }, addOpts) + ipfs.add({ path: 'download', content: filename }, addOpts) ]) expect(result.err).to.not.exist() expect(expectedResult.err).to.not.exist() @@ -299,7 +299,7 @@ module.exports = (common, options) => { const [result, expectedResult] = await Promise.all([ ipfs.add(urlSource(url), addOpts), - ipfs.add([{ path: 'download', content: Buffer.from(filename) }], addOpts) + ipfs.add([{ path: 'download', content: filename }], addOpts) ]) expect(result.err).to.not.exist() @@ -312,7 +312,7 @@ module.exports = (common, options) => { }) it('should respect raw leaves when file is smaller than one block and no metadata is present', async () => { - const file = await ipfs.add(Buffer.from([0, 1, 2]), { + const file = await ipfs.add(Uint8Array.from([0, 1, 2]), { cidVersion: 1, rawLeaves: true }) @@ -324,7 +324,7 @@ module.exports = (common, options) => { it('should override raw leaves when file is smaller than one block and metadata is present', async () => { const file = await ipfs.add({ - content: Buffer.from([0, 1, 2]), + content: Uint8Array.from([0, 1, 2]), mode: 0o123, mtime: { secs: 1000, diff --git a/packages/interface-ipfs-core/src/block/get.js b/packages/interface-ipfs-core/src/block/get.js index 7796bf6d34..7ab52f83ff 100644 --- a/packages/interface-ipfs-core/src/block/get.js +++ b/packages/interface-ipfs-core/src/block/get.js @@ -1,6 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') + +const uint8ArrayFromString = require('uint8arrays/from-string') const multihash = require('multihashing-async').multihash const CID = require('cids') const { getDescribe, getIt, expect } = require('../utils/mocha') @@ -16,7 +17,7 @@ module.exports = (common, options) => { const it = getIt(options) describe('.block.get', () => { - const data = Buffer.from('blorb') + const data = uint8ArrayFromString('blorb') let ipfs, hash before(async () => { @@ -37,19 +38,19 @@ module.exports = (common, options) => { const cid = new CID(hash) const block = await ipfs.block.get(cid) - expect(block.data).to.eql(Buffer.from('blorb')) + expect(block.data).to.eql(uint8ArrayFromString('blorb')) expect(block.cid.multihash).to.eql(cid.multihash) }) it('should get by CID in string', async () => { const block = await ipfs.block.get(multihash.toB58String(hash)) - expect(block.data).to.eql(Buffer.from('blorb')) + expect(block.data).to.eql(uint8ArrayFromString('blorb')) expect(block.cid.multihash).to.eql(hash) }) it('should get an empty block', async () => { - const res = await ipfs.block.put(Buffer.alloc(0), { + const res = await ipfs.block.put(new Uint8Array(0), { format: 'dag-pb', mhtype: 'sha2-256', version: 0 @@ -57,11 +58,11 @@ module.exports = (common, options) => { const block = await ipfs.block.get(res.cid) - expect(block.data).to.eql(Buffer.alloc(0)) + expect(block.data).to.eql(new Uint8Array(0)) }) it('should get a block added as CIDv0 with a CIDv1', async () => { - const input = Buffer.from(`TEST${Math.random()}`) + const input = uint8ArrayFromString(`TEST${Math.random()}`) const res = await ipfs.block.put(input, { version: 0 }) @@ -75,7 +76,7 @@ module.exports = (common, options) => { }) it('should get a block added as CIDv1 with a CIDv0', async () => { - const input = Buffer.from(`TEST${Math.random()}`) + const input = uint8ArrayFromString(`TEST${Math.random()}`) const res = await ipfs.block.put(input, { version: 1 }) diff --git a/packages/interface-ipfs-core/src/block/put.js b/packages/interface-ipfs-core/src/block/put.js index 475054fa2e..43a14216b1 100644 --- a/packages/interface-ipfs-core/src/block/put.js +++ b/packages/interface-ipfs-core/src/block/put.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const Block = require('ipld-block') const multihash = require('multihashing-async').multihash const CID = require('cids') @@ -28,14 +28,14 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when putting a block', () => { - return testTimeout(() => ipfs.block.put(Buffer.from('derp'), { + return testTimeout(() => ipfs.block.put(uint8ArrayFromString('derp'), { timeout: 1 })) }) it('should put a buffer, using defaults', async () => { const expectedHash = 'QmPv52ekjS75L4JmHpXVeuJ5uX2ecSfSZo88NSyxwA3rAQ' - const blob = Buffer.from('blorb') + const blob = uint8ArrayFromString('blorb') const block = await ipfs.block.put(blob) @@ -46,7 +46,7 @@ module.exports = (common, options) => { it('should put a buffer, using CID', async () => { const expectedHash = 'QmPv52ekjS75L4JmHpXVeuJ5uX2ecSfSZo88NSyxwA3rAQ' const cid = new CID(expectedHash) - const blob = Buffer.from('blorb') + const blob = uint8ArrayFromString('blorb') const block = await ipfs.block.put(blob, { cid: cid }) @@ -56,7 +56,7 @@ module.exports = (common, options) => { it('should put a buffer, using CID string', async () => { const expectedCid = 'bafyreietui4xdkiu4xvmx4fi2jivjtndbhb4drzpxomrjvd4mdz4w2avra' - const blob = Buffer.from(JSON.stringify({ hello: 'world' })) + const blob = uint8ArrayFromString(JSON.stringify({ hello: 'world' })) const block = await ipfs.block.put(blob, { cid: expectedCid }) @@ -65,7 +65,7 @@ module.exports = (common, options) => { }) it('should put a buffer, using options', async () => { - const blob = Buffer.from(`TEST${Math.random()}`) + const blob = uint8ArrayFromString(`TEST${Math.random()}`) const block = await ipfs.block.put(blob, { format: 'raw', @@ -84,16 +84,16 @@ module.exports = (common, options) => { it('should put a Block instance', async () => { const expectedHash = 'QmPv52ekjS75L4JmHpXVeuJ5uX2ecSfSZo88NSyxwA3rAQ' const cid = new CID(expectedHash) - const b = new Block(Buffer.from('blorb'), cid) + const b = new Block(uint8ArrayFromString('blorb'), cid) const block = await ipfs.block.put(b) - expect(block.data).to.eql(Buffer.from('blorb')) + expect(block.data).to.eql(uint8ArrayFromString('blorb')) expect(block.cid.multihash).to.eql(multihash.fromB58String(expectedHash)) }) it('should error with array of blocks', () => { - const blob = Buffer.from('blorb') + const blob = uint8ArrayFromString('blorb') return expect(ipfs.block.put([blob, blob])).to.eventually.be.rejected .and.be.an.instanceOf(Error) diff --git a/packages/interface-ipfs-core/src/block/rm.js b/packages/interface-ipfs-core/src/block/rm.js index 1e35c0f5da..946387e586 100644 --- a/packages/interface-ipfs-core/src/block/rm.js +++ b/packages/interface-ipfs-core/src/block/rm.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { getDescribe, getIt, expect } = require('../utils/mocha') const { nanoid } = require('nanoid') const all = require('it-all') @@ -33,7 +33,7 @@ module.exports = (common, options) => { }) it('should remove by CID object', async () => { - const cid = await ipfs.dag.put(Buffer.from(nanoid()), { + const cid = await ipfs.dag.put(uint8ArrayFromString(nanoid()), { format: 'raw', hashAlg: 'sha2-256' }) @@ -54,7 +54,7 @@ module.exports = (common, options) => { }) it('should remove by CID in string', async () => { - const cid = await ipfs.dag.put(Buffer.from(nanoid()), { + const cid = await ipfs.dag.put(uint8ArrayFromString(nanoid()), { format: 'raw', hashAlg: 'sha2-256' }) @@ -66,11 +66,11 @@ module.exports = (common, options) => { }) it('should remove by CID in buffer', async () => { - const cid = await ipfs.dag.put(Buffer.from(nanoid()), { + const cid = await ipfs.dag.put(uint8ArrayFromString(nanoid()), { format: 'raw', hashAlg: 'sha2-256' }) - const result = await all(ipfs.block.rm(cid.buffer)) + const result = await all(ipfs.block.rm(cid.bytes)) expect(result).to.be.an('array').and.to.have.lengthOf(1) expect(result[0].cid.toString()).to.equal(cid.toString()) @@ -79,15 +79,15 @@ module.exports = (common, options) => { it('should remove multiple CIDs', async () => { const cids = [ - await ipfs.dag.put(Buffer.from(nanoid()), { + await ipfs.dag.put(uint8ArrayFromString(nanoid()), { format: 'raw', hashAlg: 'sha2-256' }), - await ipfs.dag.put(Buffer.from(nanoid()), { + await ipfs.dag.put(uint8ArrayFromString(nanoid()), { format: 'raw', hashAlg: 'sha2-256' }), - await ipfs.dag.put(Buffer.from(nanoid()), { + await ipfs.dag.put(uint8ArrayFromString(nanoid()), { format: 'raw', hashAlg: 'sha2-256' }) @@ -104,7 +104,7 @@ module.exports = (common, options) => { }) it('should error when removing non-existent blocks', async () => { - const cid = await ipfs.dag.put(Buffer.from(nanoid()), { + const cid = await ipfs.dag.put(uint8ArrayFromString(nanoid()), { format: 'raw', hashAlg: 'sha2-256' }) @@ -121,7 +121,7 @@ module.exports = (common, options) => { }) it('should not error when force removing non-existent blocks', async () => { - const cid = await ipfs.dag.put(Buffer.from(nanoid()), { + const cid = await ipfs.dag.put(uint8ArrayFromString(nanoid()), { format: 'raw', hashAlg: 'sha2-256' }) @@ -138,7 +138,7 @@ module.exports = (common, options) => { }) it('should return empty output when removing blocks quietly', async () => { - const cid = await ipfs.dag.put(Buffer.from(nanoid()), { + const cid = await ipfs.dag.put(uint8ArrayFromString(nanoid()), { format: 'raw', hashAlg: 'sha2-256' }) @@ -148,7 +148,7 @@ module.exports = (common, options) => { }) it('should error when removing pinned blocks', async () => { - const cid = await ipfs.dag.put(Buffer.from(nanoid()), { + const cid = await ipfs.dag.put(uint8ArrayFromString(nanoid()), { format: 'raw', hashAlg: 'sha2-256' }) diff --git a/packages/interface-ipfs-core/src/block/stat.js b/packages/interface-ipfs-core/src/block/stat.js index b6aa745793..72d71b4203 100644 --- a/packages/interface-ipfs-core/src/block/stat.js +++ b/packages/interface-ipfs-core/src/block/stat.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const CID = require('cids') const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') @@ -16,7 +16,7 @@ module.exports = (common, options) => { const it = getIt(options) describe('.block.stat', () => { - const data = Buffer.from('blorb') + const data = uint8ArrayFromString('blorb') let ipfs, hash before(async () => { diff --git a/packages/interface-ipfs-core/src/cat.js b/packages/interface-ipfs-core/src/cat.js index e4c520ade5..6a42f4a341 100644 --- a/packages/interface-ipfs-core/src/cat.js +++ b/packages/interface-ipfs-core/src/cat.js @@ -1,10 +1,11 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require('uint8arrays/to-string') +const uint8ArrayConcat = require('uint8arrays/concat') const { fixtures } = require('./utils') const CID = require('cids') -const concat = require('it-concat') const all = require('it-all') const drain = require('it-drain') const { getDescribe, getIt, expect } = require('./utils/mocha') @@ -41,26 +42,26 @@ module.exports = (common, options) => { }) it('should cat with a base58 string encoded multihash', async () => { - const data = await concat(ipfs.cat(fixtures.smallFile.cid)) - expect(data.toString()).to.contain('Plz add me!') + const data = uint8ArrayConcat(await all(ipfs.cat(fixtures.smallFile.cid))) + expect(uint8ArrayToString(data)).to.contain('Plz add me!') }) - it('should cat with a Buffer multihash', async () => { + it('should cat with a Uint8Array multihash', async () => { const cid = new CID(fixtures.smallFile.cid).multihash - const data = await concat(ipfs.cat(cid)) - expect(data.toString()).to.contain('Plz add me!') + const data = uint8ArrayConcat(await all(ipfs.cat(cid))) + expect(uint8ArrayToString(data)).to.contain('Plz add me!') }) it('should cat with a CID object', async () => { const cid = new CID(fixtures.smallFile.cid) - const data = await concat(ipfs.cat(cid)) - expect(data.toString()).to.contain('Plz add me!') + const data = uint8ArrayConcat(await all(ipfs.cat(cid))) + expect(uint8ArrayToString(data)).to.contain('Plz add me!') }) it('should cat a file added as CIDv0 with a CIDv1', async () => { - const input = Buffer.from(`TEST${Math.random()}`) + const input = uint8ArrayFromString(`TEST${Math.random()}`) const res = await all(importer([{ content: input }], ipfs.block)) @@ -69,12 +70,12 @@ module.exports = (common, options) => { const cidv1 = cidv0.toV1() - const output = await concat(ipfs.cat(cidv1)) - expect(output.slice()).to.eql(input) + const output = uint8ArrayConcat(await all(ipfs.cat(cidv1))) + expect(output).to.eql(input) }) it('should cat a file added as CIDv1 with a CIDv0', async () => { - const input = Buffer.from(`TEST${Math.random()}`) + const input = uint8ArrayFromString(`TEST${Math.random()}`) const res = await all(importer([{ content: input }], ipfs.block, { cidVersion: 1, rawLeaves: false })) @@ -83,12 +84,12 @@ module.exports = (common, options) => { const cidv0 = cidv1.toV0() - const output = await concat(ipfs.cat(cidv0)) + const output = uint8ArrayConcat(await all(ipfs.cat(cidv0))) expect(output.slice()).to.eql(input) }) it('should cat a BIG file', async () => { - const data = await concat(ipfs.cat(fixtures.bigFile.cid)) + const data = uint8ArrayConcat(await all(ipfs.cat(fixtures.bigFile.cid))) expect(data.length).to.equal(fixtures.bigFile.data.length) expect(data.slice()).to.eql(fixtures.bigFile.data) }) @@ -96,8 +97,8 @@ module.exports = (common, options) => { it('should cat with IPFS path', async () => { const ipfsPath = '/ipfs/' + fixtures.smallFile.cid - const data = await concat(ipfs.cat(ipfsPath)) - expect(data.toString()).to.contain('Plz add me!') + const data = uint8ArrayConcat(await all(ipfs.cat(ipfsPath))) + expect(uint8ArrayToString(data)).to.contain('Plz add me!') }) it('should cat with IPFS path, nested value', async () => { @@ -108,9 +109,9 @@ module.exports = (common, options) => { const file = await filesAdded.find((f) => f.path === 'a') expect(file).to.exist() - const data = await concat(ipfs.cat(`/ipfs/${file.cid}/testfile.txt`)) + const data = uint8ArrayConcat(await all(ipfs.cat(`/ipfs/${file.cid}/testfile.txt`))) - expect(data.toString()).to.contain('Plz add me!') + expect(uint8ArrayToString(data)).to.contain('Plz add me!') }) it('should cat with IPFS path, deeply nested value', async () => { @@ -121,18 +122,18 @@ module.exports = (common, options) => { const file = filesAdded.find((f) => f.path === 'a') expect(file).to.exist() - const data = await concat(ipfs.cat(`/ipfs/${file.cid}/b/testfile.txt`)) - expect(data.toString()).to.contain('Plz add me!') + const data = uint8ArrayConcat(await all(ipfs.cat(`/ipfs/${file.cid}/b/testfile.txt`))) + expect(uint8ArrayToString(data)).to.contain('Plz add me!') }) it('should error on invalid key', () => { const invalidCid = 'somethingNotMultihash' - return expect(concat(ipfs.cat(invalidCid))).to.eventually.be.rejected() + return expect(drain(ipfs.cat(invalidCid))).to.eventually.be.rejected() }) it('should error on unknown path', () => { - return expect(concat(ipfs.cat(fixtures.smallFile.cid + '/does-not-exist'))).to.eventually.be.rejected() + return expect(drain(ipfs.cat(fixtures.smallFile.cid + '/does-not-exist'))).to.eventually.be.rejected() .and.be.an.instanceOf(Error) .and.to.have.property('message') .to.be.oneOf([ @@ -152,7 +153,7 @@ module.exports = (common, options) => { const dir = files[0] - const err = await expect(concat(ipfs.cat(dir.cid))).to.eventually.be.rejected() + const err = await expect(drain(ipfs.cat(dir.cid))).to.eventually.be.rejected() expect(err.message).to.contain('this dag node is a directory') }) @@ -160,8 +161,8 @@ module.exports = (common, options) => { const offset = 1 const length = 3 - const data = await concat(ipfs.cat(fixtures.smallFile.cid, { offset, length })) - expect(data.toString()).to.equal('lz ') + const data = uint8ArrayConcat(await all(ipfs.cat(fixtures.smallFile.cid, { offset, length }))) + expect(uint8ArrayToString(data)).to.equal('lz ') }) }) } diff --git a/packages/interface-ipfs-core/src/config/set.js b/packages/interface-ipfs-core/src/config/set.js index 2dcde164d3..82151bea6f 100644 --- a/packages/interface-ipfs-core/src/config/set.js +++ b/packages/interface-ipfs-core/src/config/set.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') @@ -81,7 +81,7 @@ module.exports = (common, options) => { }) it('should fail on non valid key', () => { - return expect(ipfs.config.set(Buffer.from('heeey'), '')).to.eventually.be.rejected() + return expect(ipfs.config.set(uint8ArrayFromString('heeey'), '')).to.eventually.be.rejected() }) it('should fail on non valid value', () => { diff --git a/packages/interface-ipfs-core/src/dag/get.js b/packages/interface-ipfs-core/src/dag/get.js index 9003706336..df9309e9b0 100644 --- a/packages/interface-ipfs-core/src/dag/get.js +++ b/packages/interface-ipfs-core/src/dag/get.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const dagPB = require('ipld-dag-pb') const DAGNode = dagPB.DAGNode const dagCBOR = require('ipld-dag-cbor') @@ -35,13 +35,13 @@ module.exports = (common, options) => { let cidCbor before(async () => { - const someData = Buffer.from('some other data') + const someData = uint8ArrayFromString('some other data') pbNode = new DAGNode(someData) cborNode = { data: someData } - nodePb = new DAGNode(Buffer.from('I am inside a Protobuf')) + nodePb = new DAGNode(uint8ArrayFromString('I am inside a Protobuf')) cidPb = await dagPB.util.cid(nodePb.serialize()) nodeCbor = { someData: 'I am inside a Cbor object', @@ -99,7 +99,7 @@ module.exports = (common, options) => { const result = await ipfs.dag.get(cidPb, { path: 'Data' }) - expect(result.value).to.eql(Buffer.from('I am inside a Protobuf')) + expect(result.value).to.eql(uint8ArrayFromString('I am inside a Protobuf')) }) it.skip('should get a dag-pb node value one level deep', (done) => {}) @@ -131,7 +131,7 @@ module.exports = (common, options) => { const result = await ipfs.dag.get(cidCbor, { path: 'pb/Data' }) - expect(result.value).to.eql(Buffer.from('I am inside a Protobuf')) + expect(result.value).to.eql(uint8ArrayFromString('I am inside a Protobuf')) }) it('should get by CID string', async () => { @@ -149,7 +149,7 @@ module.exports = (common, options) => { const cidCborStr = cidCbor.toBaseEncodedString() const result = await ipfs.dag.get(cidCborStr + '/pb/Data') - expect(result.value).to.eql(Buffer.from('I am inside a Protobuf')) + expect(result.value).to.eql(uint8ArrayFromString('I am inside a Protobuf')) }) it('should get only a CID, due to resolving locally only', async function () { @@ -166,7 +166,7 @@ module.exports = (common, options) => { }) it('should get a node added as CIDv0 with a CIDv1', async () => { - const input = Buffer.from(`TEST${Math.random()}`) + const input = uint8ArrayFromString(`TEST${Math.random()}`) const node = new DAGNode(input) @@ -180,7 +180,7 @@ module.exports = (common, options) => { }) it('should get a node added as CIDv1 with a CIDv0', async () => { - const input = Buffer.from(`TEST${Math.random()}`) + const input = uint8ArrayFromString(`TEST${Math.random()}`) const res = await all(importer([{ content: input }], ipfs.block, { cidVersion: 1, @@ -229,7 +229,7 @@ module.exports = (common, options) => { }) it('should be able to get a DAG node with format raw', async () => { - const buf = Buffer.from([0, 1, 2, 3]) + const buf = Uint8Array.from([0, 1, 2, 3]) const cid = await ipfs.dag.put(buf, { format: 'raw', diff --git a/packages/interface-ipfs-core/src/dag/put.js b/packages/interface-ipfs-core/src/dag/put.js index d247167f6a..0e22946275 100644 --- a/packages/interface-ipfs-core/src/dag/put.js +++ b/packages/interface-ipfs-core/src/dag/put.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const dagPB = require('ipld-dag-pb') const DAGNode = dagPB.DAGNode const dagCBOR = require('ipld-dag-cbor') @@ -30,7 +30,7 @@ module.exports = (common, options) => { let cborNode before((done) => { - const someData = Buffer.from('some data') + const someData = uint8ArrayFromString('some data') try { pbNode = new DAGNode(someData) @@ -90,7 +90,7 @@ module.exports = (common, options) => { expect(CID.isCID(cid)).to.equal(true) const _cid = await dagCBOR.util.cid(dagCBOR.util.serialize(cborNode)) - expect(cid.buffer).to.eql(_cid.buffer) + expect(cid.bytes).to.eql(_cid.bytes) }) it('should not fail when calling put without options', () => { diff --git a/packages/interface-ipfs-core/src/dag/resolve.js b/packages/interface-ipfs-core/src/dag/resolve.js index faab7a1585..2f5359377a 100644 --- a/packages/interface-ipfs-core/src/dag/resolve.js +++ b/packages/interface-ipfs-core/src/dag/resolve.js @@ -1,11 +1,11 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const dagPB = require('ipld-dag-pb') const DAGNode = dagPB.DAGNode const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') +const uint8ArrayFromString = require('uint8arrays/from-string') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -105,7 +105,7 @@ module.exports = (common, options) => { }) it('should resolve a raw node', async () => { - const node = Buffer.from(['hello world']) + const node = Uint8Array.from(['hello world']) const cid = await ipfs.dag.put(node, { format: 'raw', hashAlg: 'sha2-256' }) const result = await ipfs.dag.resolve(cid, { path: '/' }) @@ -114,12 +114,12 @@ module.exports = (common, options) => { }) it('should resolve a path inside a dag-pb node linked to from another dag-pb node', async () => { - const someData = Buffer.from('some other data') + const someData = uint8ArrayFromString('some other data') const childNode = new DAGNode(someData) const childCid = await ipfs.dag.put(childNode, { format: 'dag-pb', hashAlg: 'sha2-256' }) const linkToChildNode = await childNode.toDAGLink({ name: 'foo', cidVersion: 0 }) - const parentNode = new DAGNode(Buffer.from('derp'), [linkToChildNode]) + const parentNode = new DAGNode(uint8ArrayFromString('derp'), [linkToChildNode]) const parentCid = await ipfs.dag.put(parentNode, { format: 'dag-pb', hashAlg: 'sha2-256' }) const result = await ipfs.dag.resolve(parentCid, { path: '/foo' }) diff --git a/packages/interface-ipfs-core/src/dag/tree.js b/packages/interface-ipfs-core/src/dag/tree.js index ce1393da6b..9bee426413 100644 --- a/packages/interface-ipfs-core/src/dag/tree.js +++ b/packages/interface-ipfs-core/src/dag/tree.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const dagPB = require('ipld-dag-pb') const DAGNode = dagPB.DAGNode const dagCBOR = require('ipld-dag-cbor') @@ -33,7 +33,7 @@ module.exports = (common, options) => { let cidCbor before(async function () { - nodePb = new DAGNode(Buffer.from('I am inside a Protobuf')) + nodePb = new DAGNode(uint8ArrayFromString('I am inside a Protobuf')) cidPb = await dagPB.util.cid(nodePb.serialize()) nodeCbor = { diff --git a/packages/interface-ipfs-core/src/dht/get.js b/packages/interface-ipfs-core/src/dht/get.js index 2a02c26d44..7ff2f1284b 100644 --- a/packages/interface-ipfs-core/src/dht/get.js +++ b/packages/interface-ipfs-core/src/dht/get.js @@ -3,6 +3,7 @@ const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') +const uint8ArrayToString = require('uint8arrays/to-string') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -44,7 +45,7 @@ module.exports = (common, options) => { const publish = await nodeA.name.publish(data.cid) const record = await nodeA.dht.get(`/ipns/${publish.name}`) - expect(record.toString()).to.contain(data.cid.toString()) + expect(uint8ArrayToString(record)).to.contain(data.cid.toString()) }) }) } diff --git a/packages/interface-ipfs-core/src/dht/provide.js b/packages/interface-ipfs-core/src/dht/provide.js index 425e37f960..2b729f45d4 100644 --- a/packages/interface-ipfs-core/src/dht/provide.js +++ b/packages/interface-ipfs-core/src/dht/provide.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const CID = require('cids') const all = require('it-all') const { getDescribe, getIt, expect } = require('../utils/mocha') @@ -30,7 +30,7 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when providing a value on the DHT', async () => { - const res = await ipfs.add(Buffer.from('test')) + const res = await ipfs.add(uint8ArrayFromString('test')) await testTimeout(() => ipfs.dht.provide(res.cid, { timeout: 1 @@ -38,7 +38,7 @@ module.exports = (common, options) => { }) it('should provide local CID', async () => { - const res = await ipfs.add(Buffer.from('test')) + const res = await ipfs.add(uint8ArrayFromString('test')) await all(ipfs.dht.provide(res.cid)) }) @@ -54,15 +54,15 @@ module.exports = (common, options) => { it('should allow multiple CIDs to be passed', async () => { const res = await all(ipfs.addAll([ - { content: Buffer.from('t0') }, - { content: Buffer.from('t1') } + { content: uint8ArrayFromString('t0') }, + { content: uint8ArrayFromString('t1') } ])) await all(ipfs.dht.provide(res.map(f => f.cid))) }) it('should provide a CIDv1', async () => { - const res = await ipfs.add(Buffer.from('test'), { cidVersion: 1 }) + const res = await ipfs.add(uint8ArrayFromString('test'), { cidVersion: 1 }) await all(ipfs.dht.provide(res.cid)) }) diff --git a/packages/interface-ipfs-core/src/dht/utils.js b/packages/interface-ipfs-core/src/dht/utils.js index ea1c71773e..96a391d989 100644 --- a/packages/interface-ipfs-core/src/dht/utils.js +++ b/packages/interface-ipfs-core/src/dht/utils.js @@ -1,11 +1,11 @@ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const multihashing = require('multihashing-async') const CID = require('cids') exports.fakeCid = async (data) => { - const bytes = data || Buffer.from(`TEST${Math.random()}`) + const bytes = data || uint8ArrayFromString(`TEST${Math.random()}`) const mh = await multihashing(bytes, 'sha2-256') return new CID(0, 'dag-pb', mh) } diff --git a/packages/interface-ipfs-core/src/files/chmod.js b/packages/interface-ipfs-core/src/files/chmod.js index 0b501fcf81..0705d41731 100644 --- a/packages/interface-ipfs-core/src/files/chmod.js +++ b/packages/interface-ipfs-core/src/files/chmod.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { nanoid } = require('nanoid') const { getDescribe, getIt, expect } = require('../utils/mocha') const isShardAtPath = require('../utils/is-shard-at-path') @@ -19,7 +19,7 @@ module.exports = (common, options) => { async function testChmod (initialMode, modification, expectedFinalMode) { const path = `/test-${nanoid()}` - await ipfs.files.write(path, Buffer.from('Hello world!'), { + await ipfs.files.write(path, uint8ArrayFromString('Hello world!'), { create: true, mtime: new Date(), mode: initialMode @@ -41,7 +41,7 @@ module.exports = (common, options) => { it('should update the mode for a file', async () => { const path = `/foo-${Math.random()}` - await ipfs.files.write(path, Buffer.from('Hello world'), { + await ipfs.files.write(path, uint8ArrayFromString('Hello world'), { create: true, mtime: new Date() }) @@ -73,7 +73,7 @@ module.exports = (common, options) => { const path = `/foo-${Math.random()}` await ipfs.files.mkdir(path) - await ipfs.files.write(`${path}/foo.txt`, Buffer.from('Hello world'), { + await ipfs.files.write(`${path}/foo.txt`, uint8ArrayFromString('Hello world'), { create: true, shardSplitThreshold: 0 }) @@ -335,7 +335,7 @@ module.exports = (common, options) => { it('should respect timeout option when changing the mode of a file', async () => { const path = `/foo-${Math.random()}` - await ipfs.files.write(path, Buffer.from('Hello world'), { + await ipfs.files.write(path, uint8ArrayFromString('Hello world'), { create: true }) diff --git a/packages/interface-ipfs-core/src/files/cp.js b/packages/interface-ipfs-core/src/files/cp.js index 2fffabba5c..347006b52e 100644 --- a/packages/interface-ipfs-core/src/files/cp.js +++ b/packages/interface-ipfs-core/src/files/cp.js @@ -1,10 +1,10 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayConcat = require('uint8arrays/concat') const { nanoid } = require('nanoid') const all = require('it-all') -const concat = require('it-concat') const { fixtures } = require('../utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const mh = require('multihashing-async').multihash @@ -73,8 +73,8 @@ module.exports = (common, options) => { const src1 = `/src2-${Math.random()}` const parent = `/output-${Math.random()}` - const cid = new CID(1, 'identity', mh.encode(Buffer.from('derp'), 'identity')) - await ipfs.block.put(new Block(Buffer.from('derp'), cid), { cid }) + const cid = new CID(1, 'identity', mh.encode(uint8ArrayFromString('derp'), 'identity')) + await ipfs.block.put(new Block(uint8ArrayFromString('derp'), cid), { cid }) await ipfs.files.cp(`/ipfs/${cid}`, parent) await ipfs.files.write(src1, [], { @@ -129,9 +129,9 @@ module.exports = (common, options) => { await ipfs.files.cp(source, destination) - const buffer = await concat(ipfs.files.read(destination)) + const bytes = uint8ArrayConcat(await all(ipfs.files.read(destination))) - expect(buffer.slice()).to.deep.equal(data) + expect(bytes).to.deep.equal(data) }) it('copies a file to a pre-existing directory', async () => { @@ -199,9 +199,9 @@ module.exports = (common, options) => { }) for (const source of sources) { - const buffer = await concat(ipfs.files.read(`${destination}${source.path}`)) + const bytes = uint8ArrayConcat(await all(ipfs.files.read(`${destination}${source.path}`))) - expect(buffer.slice()).to.deep.equal(source.data) + expect(bytes).to.deep.equal(source.data) } }) @@ -311,7 +311,7 @@ module.exports = (common, options) => { const filePath = `/${file}` const finalFilePath = `${shardedDirPath}/${file}` - await ipfs.files.write(filePath, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(filePath, Uint8Array.from([0, 1, 2, 3]), { create: true }) @@ -331,7 +331,7 @@ module.exports = (common, options) => { const filePath = `${shardedDirPath}/${file}` const finalFilePath = `${othershardedDirPath}/${file}` - await ipfs.files.write(filePath, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(filePath, Uint8Array.from([0, 1, 2, 3]), { create: true }) @@ -354,7 +354,7 @@ module.exports = (common, options) => { const filePath = `${shardedDirPath}/${file}` const finalFilePath = `${dirPath}/${file}` - await ipfs.files.write(filePath, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(filePath, Uint8Array.from([0, 1, 2, 3]), { create: true }) @@ -380,7 +380,7 @@ module.exports = (common, options) => { nsecs: (mtime - (seconds * 1000)) * 1000 } - await ipfs.files.write(testSrcPath, Buffer.from('TEST'), { + await ipfs.files.write(testSrcPath, uint8ArrayFromString('TEST'), { create: true, mode, mtime diff --git a/packages/interface-ipfs-core/src/files/ls.js b/packages/interface-ipfs-core/src/files/ls.js index 1fc930eb3c..031075f2af 100644 --- a/packages/interface-ipfs-core/src/files/ls.js +++ b/packages/interface-ipfs-core/src/files/ls.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { getDescribe, getIt, expect } = require('../utils/mocha') const CID = require('cids') const createShardedDirectory = require('../utils/create-sharded-directory') @@ -41,7 +41,7 @@ module.exports = (common, options) => { it('lists the root directory', async () => { const fileName = `small-file-${Math.random()}.txt` - const content = Buffer.from('Hello world') + const content = uint8ArrayFromString('Hello world') await ipfs.files.write(`/${fileName}`, content, { create: true @@ -68,7 +68,7 @@ module.exports = (common, options) => { it('lists files in a directory', async () => { const dirName = `dir-${Math.random()}` const fileName = `small-file-${Math.random()}.txt` - const content = Buffer.from('Hello world') + const content = uint8ArrayFromString('Hello world') await ipfs.files.write(`/${dirName}/${fileName}`, content, { create: true, @@ -87,7 +87,7 @@ module.exports = (common, options) => { it('lists a file', async () => { const fileName = `small-file-${Math.random()}.txt` - const content = Buffer.from('Hello world') + const content = uint8ArrayFromString('Hello world') await ipfs.files.write(`/${fileName}`, content, { create: true @@ -194,7 +194,7 @@ module.exports = (common, options) => { const fileName = `small-file-${Math.random()}.txt` await ipfs.files.mkdir(`${dirPath}`) - await ipfs.files.write(`${dirPath}/${fileName}`, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(`${dirPath}/${fileName}`, Uint8Array.from([0, 1, 2, 3]), { create: true }) diff --git a/packages/interface-ipfs-core/src/files/mv.js b/packages/interface-ipfs-core/src/files/mv.js index f0fd539841..eb4d13735a 100644 --- a/packages/interface-ipfs-core/src/files/mv.js +++ b/packages/interface-ipfs-core/src/files/mv.js @@ -1,13 +1,14 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayConcat = require('uint8arrays/concat') const { getDescribe, getIt, expect } = require('../utils/mocha') const createShardedDirectory = require('../utils/create-sharded-directory') -const concat = require('it-concat') const randomBytes = require('iso-random-stream/src/random') const isShardAtPath = require('../utils/is-shard-at-path') const testTimeout = require('../utils/test-timeout') +const all = require('it-all') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -27,7 +28,7 @@ module.exports = (common, options) => { before(async () => { await ipfs.files.mkdir('/test/lv1/lv2', { parents: true }) - await ipfs.files.write('/test/a', Buffer.from('Hello, world!'), { create: true }) + await ipfs.files.write('/test/a', uint8ArrayFromString('Hello, world!'), { create: true }) }) after(() => common.clean()) @@ -49,8 +50,8 @@ module.exports = (common, options) => { }) await ipfs.files.mv(source, destination) - const buffer = await concat(ipfs.files.read(destination)) - expect(buffer.slice()).to.deep.equal(data) + const bytes = uint8ArrayConcat(await all(ipfs.files.read(destination))) + expect(bytes).to.deep.equal(data) await expect(ipfs.files.stat(source)).to.eventually.be.rejectedWith(/does not exist/) }) @@ -170,7 +171,7 @@ module.exports = (common, options) => { const finalFilePath = `${shardedDirPath}/${file}` await ipfs.files.mkdir(dirPath) - await ipfs.files.write(filePath, Buffer.from([0, 1, 2, 3, 4]), { + await ipfs.files.write(filePath, Uint8Array.from([0, 1, 2, 3, 4]), { create: true }) @@ -196,7 +197,7 @@ module.exports = (common, options) => { const finalFilePath = `${dirPath}/${file}` await ipfs.files.mkdir(dirPath) - await ipfs.files.write(filePath, Buffer.from([0, 1, 2, 3, 4]), { + await ipfs.files.write(filePath, Uint8Array.from([0, 1, 2, 3, 4]), { create: true }) @@ -222,7 +223,7 @@ module.exports = (common, options) => { const filePath = `${shardedDirPath}/${file}` const finalFilePath = `${otherShardedDirPath}/${file}` - await ipfs.files.write(filePath, Buffer.from([0, 1, 2, 3, 4]), { + await ipfs.files.write(filePath, Uint8Array.from([0, 1, 2, 3, 4]), { create: true }) @@ -249,7 +250,7 @@ module.exports = (common, options) => { const filePath = `${shardedDirPath}/${file}` const finalFilePath = `${otherShardedDirPath}/${file}` - await ipfs.files.write(filePath, Buffer.from([0, 1, 2, 3, 4]), { + await ipfs.files.write(filePath, Uint8Array.from([0, 1, 2, 3, 4]), { create: true }) diff --git a/packages/interface-ipfs-core/src/files/read.js b/packages/interface-ipfs-core/src/files/read.js index baa313143e..038d89351a 100644 --- a/packages/interface-ipfs-core/src/files/read.js +++ b/packages/interface-ipfs-core/src/files/read.js @@ -1,9 +1,9 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') -const concat = require('it-concat') +const uint8ArrayConcat = require('uint8arrays/concat') const drain = require('it-drain') +const all = require('it-all') const { fixtures } = require('../utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const createShardedDirectory = require('../utils/create-sharded-directory') @@ -36,9 +36,9 @@ module.exports = (common, options) => { create: true }) - const buffer = await concat(ipfs.files.read(filePath)) + const bytes = uint8ArrayConcat(await all(ipfs.files.read(filePath))) - expect(buffer.slice()).to.deep.equal(smallFile) + expect(bytes).to.deep.equal(smallFile) }) it('reads a file with an offset', async () => { @@ -50,11 +50,11 @@ module.exports = (common, options) => { create: true }) - const buffer = await concat(ipfs.files.read(path, { + const bytes = uint8ArrayConcat(await all(ipfs.files.read(path, { offset - })) + }))) - expect(buffer.slice()).to.deep.equal(data.slice(offset)) + expect(bytes).to.deep.equal(data.slice(offset)) }) it('reads a file with a length', async () => { @@ -66,11 +66,11 @@ module.exports = (common, options) => { create: true }) - const buffer = await concat(ipfs.files.read(path, { + const bytes = uint8ArrayConcat(await all(ipfs.files.read(path, { length - })) + }))) - expect(buffer.slice()).to.deep.equal(data.slice(0, length)) + expect(bytes).to.deep.equal(data.slice(0, length)) }) it('reads a file with a legacy count argument', async () => { @@ -82,11 +82,11 @@ module.exports = (common, options) => { create: true }) - const buffer = await concat(ipfs.files.read(path, { + const buffer = uint8ArrayConcat(await all(ipfs.files.read(path, { count: length - })) + }))) - expect(buffer.slice()).to.deep.equal(data.slice(0, length)) + expect(buffer).to.deep.equal(data.slice(0, length)) }) it('reads a file with an offset and a length', async () => { @@ -99,12 +99,12 @@ module.exports = (common, options) => { create: true }) - const buffer = await concat(ipfs.files.read(path, { + const buffer = uint8ArrayConcat(await all(ipfs.files.read(path, { offset, length - })) + }))) - expect(buffer.slice()).to.deep.equal(data.slice(offset, offset + length)) + expect(buffer).to.deep.equal(data.slice(offset, offset + length)) }) it('reads a file with an offset and a legacy count argument', async () => { @@ -117,44 +117,44 @@ module.exports = (common, options) => { create: true }) - const buffer = await concat(ipfs.files.read(path, { + const buffer = uint8ArrayConcat(await all(ipfs.files.read(path, { offset, count: length - })) + }))) - expect(buffer.slice()).to.deep.equal(data.slice(offset, offset + length)) + expect(buffer).to.deep.equal(data.slice(offset, offset + length)) }) it('refuses to read a directory', async () => { const path = '/' - await expect(concat(ipfs.files.read(path))).to.eventually.be.rejectedWith(/not a file/) + await expect(drain(ipfs.files.read(path))).to.eventually.be.rejectedWith(/not a file/) }) it('refuses to read a non-existent file', async () => { const path = `/file-${Math.random()}.txt` - await expect(concat(ipfs.files.read(path))).to.eventually.be.rejectedWith(/does not exist/) + await expect(drain(ipfs.files.read(path))).to.eventually.be.rejectedWith(/does not exist/) }) it('reads file from inside a sharded directory', async () => { const shardedDirPath = await createShardedDirectory(ipfs) const filePath = `${shardedDirPath}/file-${Math.random()}.txt` - const content = Buffer.from([0, 1, 2, 3, 4]) + const content = Uint8Array.from([0, 1, 2, 3, 4]) await ipfs.files.write(filePath, content, { create: true }) - const buffer = await concat(ipfs.files.read(filePath)) + const bytes = uint8ArrayConcat(await all(ipfs.files.read(filePath))) - expect(buffer.slice()).to.deep.equal(content) + expect(bytes).to.deep.equal(content) }) it('should read from outside of mfs', async () => { const { cid } = await ipfs.add(fixtures.smallFile.data) - const testFileData = await concat(ipfs.files.read(`/ipfs/${cid}`)) - expect(testFileData.slice()).to.eql(fixtures.smallFile.data) + const testFileData = uint8ArrayConcat(await all(ipfs.files.read(`/ipfs/${cid}`))) + expect(testFileData).to.eql(fixtures.smallFile.data) }) it('should respect timeout option when reading files', async () => { diff --git a/packages/interface-ipfs-core/src/files/stat.js b/packages/interface-ipfs-core/src/files/stat.js index ab96ee337d..46e4b0e6ae 100644 --- a/packages/interface-ipfs-core/src/files/stat.js +++ b/packages/interface-ipfs-core/src/files/stat.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { nanoid } = require('nanoid') const { fixtures } = require('../utils') const { getDescribe, getIt, expect } = require('../utils/mocha') @@ -187,7 +187,7 @@ module.exports = (common, options) => { }) it('stats an identity CID', async () => { - const data = Buffer.from('derp') + const data = uint8ArrayFromString('derp') const path = `/test-${nanoid()}/identity.node` const cid = new CID(1, 'identity', mh.encode(data, 'identity')) await ipfs.block.put(new Block(data, cid)) @@ -205,7 +205,7 @@ module.exports = (common, options) => { const testDir = `/test-${nanoid()}` await ipfs.files.mkdir(testDir, { parents: true }) - await ipfs.files.write(`${testDir}/b`, Buffer.from('Hello, world!'), { create: true }) + await ipfs.files.write(`${testDir}/b`, uint8ArrayFromString('Hello, world!'), { create: true }) const stat = await ipfs.files.stat(`${testDir}/b`) @@ -218,7 +218,7 @@ module.exports = (common, options) => { const testDir = `/test-${nanoid()}` await ipfs.files.mkdir(testDir, { parents: true }) - await ipfs.files.write(`${testDir}/b`, Buffer.from('Hello, world!'), { + await ipfs.files.write(`${testDir}/b`, uint8ArrayFromString('Hello, world!'), { create: true, mtime: { secs: 5, @@ -240,7 +240,7 @@ module.exports = (common, options) => { const testDir = `/test-${nanoid()}` await ipfs.files.mkdir(testDir, { parents: true }) - await ipfs.files.write(`${testDir}/a`, Buffer.from('Hello, world!'), { create: true }) + await ipfs.files.write(`${testDir}/a`, uint8ArrayFromString('Hello, world!'), { create: true }) const stat = await ipfs.files.stat(testDir) stat.cid = stat.cid.toString() @@ -291,7 +291,7 @@ module.exports = (common, options) => { const testDir = `/test-${nanoid()}` await ipfs.files.mkdir(testDir, { parents: true }) - await ipfs.files.write(`${testDir}/a`, Buffer.from('Hello, world!'), { + await ipfs.files.write(`${testDir}/a`, uint8ArrayFromString('Hello, world!'), { create: true, shardSplitThreshold: 0 }) @@ -315,7 +315,7 @@ module.exports = (common, options) => { nsecs: 0 } }) - await ipfs.files.write(`${testDir}/a`, Buffer.from('Hello, world!'), { + await ipfs.files.write(`${testDir}/a`, uint8ArrayFromString('Hello, world!'), { create: true, shardSplitThreshold: 0 }) diff --git a/packages/interface-ipfs-core/src/files/touch.js b/packages/interface-ipfs-core/src/files/touch.js index b229df99f2..062d7ad1e0 100644 --- a/packages/interface-ipfs-core/src/files/touch.js +++ b/packages/interface-ipfs-core/src/files/touch.js @@ -1,12 +1,13 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayConcat = require('uint8arrays/concat') const { nanoid } = require('nanoid') const { getDescribe, getIt, expect } = require('../utils/mocha') const delay = require('delay') -const concat = require('it-concat') const testTimeout = require('../utils/test-timeout') +const all = require('it-all') module.exports = (common, options) => { const describe = getDescribe(options) @@ -20,7 +21,7 @@ module.exports = (common, options) => { async function testMtime (mtime, expectedMtime) { const testPath = `/test-${nanoid()}` - await ipfs.files.write(testPath, Buffer.from('Hello, world!'), { + await ipfs.files.write(testPath, uint8ArrayFromString('Hello, world!'), { create: true }) @@ -43,7 +44,7 @@ module.exports = (common, options) => { this.slow(5 * 1000) const testPath = `/test-${nanoid()}` - await ipfs.files.write(testPath, Buffer.from('Hello, world!'), { + await ipfs.files.write(testPath, uint8ArrayFromString('Hello, world!'), { create: true }) @@ -65,7 +66,7 @@ module.exports = (common, options) => { const mtime = new Date() const seconds = Math.floor(mtime.getTime() / 1000) - await ipfs.files.write(testPath, Buffer.from('Hello, world!'), { + await ipfs.files.write(testPath, uint8ArrayFromString('Hello, world!'), { create: true, mtime }) @@ -99,7 +100,7 @@ module.exports = (common, options) => { await ipfs.files.mkdir(path, { mtime: new Date() }) - await ipfs.files.write(`${path}/foo.txt`, Buffer.from('Hello world'), { + await ipfs.files.write(`${path}/foo.txt`, uint8ArrayFromString('Hello world'), { create: true, shardSplitThreshold: 0 }) @@ -120,9 +121,9 @@ module.exports = (common, options) => { flush: true }) - const buffer = await concat(ipfs.files.read(path)) + const bytes = uint8ArrayConcat(await all(ipfs.files.read(path))) - expect(buffer.slice()).to.deep.equal(Buffer.from([])) + expect(bytes.slice()).to.deep.equal(Uint8Array.from([])) }) it('should set mtime as Date', async function () { diff --git a/packages/interface-ipfs-core/src/files/write.js b/packages/interface-ipfs-core/src/files/write.js index fb48247ae3..c30f152355 100644 --- a/packages/interface-ipfs-core/src/files/write.js +++ b/packages/interface-ipfs-core/src/files/write.js @@ -1,7 +1,8 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayConcat = require('uint8arrays/concat') const { nanoid } = require('nanoid') const { getDescribe, getIt, expect } = require('../utils/mocha') const { isNode } = require('ipfs-utils/src/env') @@ -12,7 +13,6 @@ const createTwoShards = require('../utils/create-two-shards') const randomBytes = require('iso-random-stream/src/random') const randomStream = require('iso-random-stream') const all = require('it-all') -const concat = require('it-concat') const isShardAtPath = require('../utils/is-shard-at-path') const testTimeout = require('../utils/test-timeout') @@ -65,7 +65,7 @@ module.exports = (common, options) => { async function testMode (mode, expectedMode) { const testPath = `/test-${nanoid()}` - await ipfs.files.write(testPath, Buffer.from('Hello, world!'), { + await ipfs.files.write(testPath, uint8ArrayFromString('Hello, world!'), { create: true, parents: true, mode @@ -78,7 +78,7 @@ module.exports = (common, options) => { async function testMtime (mtime, expectedMtime) { const testPath = `/test-${nanoid()}` - await ipfs.files.write(testPath, Buffer.from('Hello, world!'), { + await ipfs.files.write(testPath, uint8ArrayFromString('Hello, world!'), { create: true, parents: true, mtime @@ -107,20 +107,20 @@ module.exports = (common, options) => { }) it('explodes if given a negtive offset', async () => { - await expect(ipfs.files.write('/foo-negative-offset', Buffer.from('foo'), { + await expect(ipfs.files.write('/foo-negative-offset', uint8ArrayFromString('foo'), { offset: -1 })).to.eventually.be.rejected() }) it('explodes if given a negative length', async () => { - await expect(ipfs.files.write('/foo-negative-length', Buffer.from('foo'), { + await expect(ipfs.files.write('/foo-negative-length', uint8ArrayFromString('foo'), { length: -1 })).to.eventually.be.rejected() }) it('creates a zero length file when passed a zero length', async () => { const path = '/foo-zero-length' - await ipfs.files.write(path, Buffer.from('foo'), { + await ipfs.files.write(path, uint8ArrayFromString('foo'), { length: 0, create: true }) @@ -140,7 +140,7 @@ module.exports = (common, options) => { }) await expect(ipfs.files.stat(filePath)).to.eventually.have.property('size', smallFile.length) - expect(Buffer.concat(await all(ipfs.files.read(filePath)))).to.deep.equal(smallFile) + expect(uint8ArrayConcat(await all(ipfs.files.read(filePath)))).to.deep.equal(smallFile) }) it('writes a small file using a string', async function () { @@ -152,7 +152,7 @@ module.exports = (common, options) => { }) await expect(ipfs.files.stat(filePath)).to.eventually.have.property('size', content.length) - expect(Buffer.concat(await all(ipfs.files.read(filePath)))).to.deep.equal(Buffer.from(content)) + expect(uint8ArrayConcat(await all(ipfs.files.read(filePath)))).to.deep.equal(uint8ArrayFromString(content)) }) it('writes part of a small file using a string', async function () { @@ -256,12 +256,12 @@ module.exports = (common, options) => { it('refuses to write to a path that has a file in it', async () => { const filePath = `/small-file-${Math.random()}.txt` - await ipfs.files.write(filePath, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(filePath, Uint8Array.from([0, 1, 2, 3]), { create: true }) try { - await ipfs.files.write(`${filePath}/other-file-${Math.random()}.txt`, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(`${filePath}/other-file-${Math.random()}.txt`, Uint8Array.from([0, 1, 2, 3]), { create: true }) @@ -279,15 +279,15 @@ module.exports = (common, options) => { length: 2 }) - const buffer = await concat(ipfs.files.read(path)) + const bytes = uint8ArrayConcat(await all(ipfs.files.read(path))) - expect(buffer.length).to.equal(2) + expect(bytes.length).to.equal(2) }) }) runTest(({ type, path, content, contentSize }) => { it(`overwrites start of a file without truncating (${type})`, async () => { - const newContent = Buffer.from('Goodbye world') + const newContent = uint8ArrayFromString('Goodbye world') await ipfs.files.write(path, content, { create: true @@ -300,7 +300,7 @@ module.exports = (common, options) => { const stats = await ipfs.files.stat(path) expect(stats.size).to.equal(contentSize) - const buffer = Buffer.concat(await all(ipfs.files.read(path, { + const buffer = uint8ArrayConcat(await all(ipfs.files.read(path, { offset: 0, length: newContent.length }))) @@ -320,19 +320,19 @@ module.exports = (common, options) => { await expect(ipfs.files.stat(path)).to.eventually.have.property('size', offset + contentSize) - const buffer = Buffer.concat(await all(ipfs.files.read(path, { + const buffer = uint8ArrayConcat(await all(ipfs.files.read(path, { offset: 0, length: offset }))) - expect(buffer).to.deep.equal(Buffer.alloc(offset, 0)) + expect(buffer).to.deep.equal(new Uint8Array(offset, 0)) }) }) runTest(({ type, path, content, contentSize }) => { it(`expands a file when an offset is specified (${type})`, async () => { const offset = contentSize - 1 - const newContent = Buffer.from('Oh hai!') + const newContent = uint8ArrayFromString('Oh hai!') await ipfs.files.write(path, content, { create: true @@ -344,7 +344,7 @@ module.exports = (common, options) => { await expect(ipfs.files.stat(path)).to.eventually.have.property('size', contentSize + newContent.length - 1) - const buffer = Buffer.concat(await all(ipfs.files.read(path, { + const buffer = uint8ArrayConcat(await all(ipfs.files.read(path, { offset: offset }))) @@ -355,7 +355,7 @@ module.exports = (common, options) => { runTest(({ type, path, content, contentSize }) => { it(`expands a file when an offset is specified and the offset is longer than the file (${type})`, async () => { const offset = contentSize + 5 - const newContent = Buffer.from('Oh hai!') + const newContent = uint8ArrayFromString('Oh hai!') await ipfs.files.write(path, content, { create: true @@ -366,19 +366,19 @@ module.exports = (common, options) => { await expect(ipfs.files.stat(path)).to.eventually.have.property('size', newContent.length + offset) - const buffer = Buffer.concat(await all(ipfs.files.read(path))) + const buffer = uint8ArrayConcat(await all(ipfs.files.read(path))) if (content[Symbol.asyncIterator]) { - content = Buffer.concat(await all(content)) + content = uint8ArrayConcat(await all(content)) } - expect(buffer).to.deep.equal(Buffer.concat([content, Buffer.from([0, 0, 0, 0, 0]), newContent])) + expect(buffer).to.deep.equal(uint8ArrayConcat([content, Uint8Array.from([0, 0, 0, 0, 0]), newContent])) }) }) runTest(({ type, path, content }) => { it(`truncates a file after writing (${type})`, async () => { - const newContent = Buffer.from('Oh hai!') + const newContent = uint8ArrayFromString('Oh hai!') await ipfs.files.write(path, content, { create: true @@ -389,7 +389,7 @@ module.exports = (common, options) => { await expect(ipfs.files.stat(path)).to.eventually.have.property('size', newContent.length) - const buffer = Buffer.concat(await all(ipfs.files.read(path))) + const buffer = uint8ArrayConcat(await all(ipfs.files.read(path))) expect(buffer).to.deep.equal(newContent) }) @@ -450,7 +450,7 @@ module.exports = (common, options) => { offset: 0 }) - const actualBytes = Buffer.concat(await all(ipfs.files.read(fileName))) + const actualBytes = uint8ArrayConcat(await all(ipfs.files.read(fileName))) for (var i = 0; i < newDataStream.length; i++) { if (newDataStream[i] !== actualBytes[i]) { @@ -476,7 +476,7 @@ module.exports = (common, options) => { }) for (let i = 0; i < shardSplitThreshold; i++) { - await ipfs.files.write(`/${dirPath}/file-${Math.random()}`, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(`/${dirPath}/file-${Math.random()}`, Uint8Array.from([0, 1, 2, 3]), { create: true, shardSplitThreshold }) @@ -484,7 +484,7 @@ module.exports = (common, options) => { await expect(ipfs.files.stat(dirPath)).to.eventually.have.property('type', 'directory') - await ipfs.files.write(newFilePath, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(newFilePath, Uint8Array.from([0, 1, 2, 3]), { create: true, shardSplitThreshold }) @@ -506,7 +506,7 @@ module.exports = (common, options) => { const newFile = `file-${Math.random()}` const newFilePath = `${shardedDirPath}/${newFile}` - await ipfs.files.write(newFilePath, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(newFilePath, Uint8Array.from([0, 1, 2, 3]), { create: true }) @@ -531,9 +531,9 @@ module.exports = (common, options) => { const shardedDirPath = await createShardedDirectory(ipfs) const newFile = 'file-0.6944395883502592' const newFilePath = `${shardedDirPath}/${newFile}` - const newContent = Buffer.from([3, 2, 1, 0]) + const newContent = Uint8Array.from([3, 2, 1, 0]) - await ipfs.files.write(newFilePath, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(newFilePath, Uint8Array.from([0, 1, 2, 3]), { create: true }) @@ -547,7 +547,7 @@ module.exports = (common, options) => { }) // read the file back - const buffer = Buffer.concat(await all(ipfs.files.read(newFilePath))) + const buffer = uint8ArrayConcat(await all(ipfs.files.read(newFilePath))) expect(buffer).to.deep.equal(newContent) @@ -561,9 +561,9 @@ module.exports = (common, options) => { const shardedDirPath = await createShardedDirectory(ipfs) const newFile = `file-${Math.random()}` const newFilePath = `${shardedDirPath}/${newFile}` - const newContent = Buffer.from([3, 2, 1, 0]) + const newContent = Uint8Array.from([3, 2, 1, 0]) - await ipfs.files.write(newFilePath, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(newFilePath, Uint8Array.from([0, 1, 2, 3]), { create: true }) @@ -577,7 +577,7 @@ module.exports = (common, options) => { }) // read the file back - const buffer = Buffer.concat(await all(ipfs.files.read(newFilePath))) + const buffer = uint8ArrayConcat(await all(ipfs.files.read(newFilePath))) expect(buffer).to.deep.equal(newContent) @@ -591,9 +591,9 @@ module.exports = (common, options) => { const shardedDirPath = await createShardedDirectory(ipfs) const newFile = 'file-1a.txt' const newFilePath = `${shardedDirPath}/${newFile}` - const newContent = Buffer.from([3, 2, 1, 0]) + const newContent = Uint8Array.from([3, 2, 1, 0]) - await ipfs.files.write(newFilePath, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(newFilePath, Uint8Array.from([0, 1, 2, 3]), { create: true }) @@ -607,7 +607,7 @@ module.exports = (common, options) => { }) // read the file back - const buffer = Buffer.concat(await all(ipfs.files.read(newFilePath))) + const buffer = uint8ArrayConcat(await all(ipfs.files.read(newFilePath))) expect(buffer).to.deep.equal(newContent) @@ -622,7 +622,7 @@ module.exports = (common, options) => { const directoryPath = `/${directory}` const fileName = `file-${Math.random()}.txt` const filePath = `${directoryPath}/${fileName}` - const expectedBytes = Buffer.from([0, 1, 2, 3]) + const expectedBytes = Uint8Array.from([0, 1, 2, 3]) await ipfs.files.mkdir(directoryPath, { cidVersion: 0 @@ -637,7 +637,7 @@ module.exports = (common, options) => { await expect(ipfs.files.stat(filePath)).to.eventually.have.nested.property('cid.version', 1) - const actualBytes = Buffer.concat(await all(ipfs.files.read(filePath))) + const actualBytes = uint8ArrayConcat(await all(ipfs.files.read(filePath))) expect(actualBytes).to.deep.equal(expectedBytes) }) @@ -647,7 +647,7 @@ module.exports = (common, options) => { const directoryPath = `/${directory}` const fileName = `file-${Math.random()}.txt` const filePath = `${directoryPath}/${fileName}` - const expectedBytes = Buffer.from([0, 1, 2, 3]) + const expectedBytes = Uint8Array.from([0, 1, 2, 3]) await ipfs.files.mkdir(directoryPath, { cidVersion: 0 @@ -655,7 +655,7 @@ module.exports = (common, options) => { await expect(ipfs.files.stat(directoryPath)).to.eventually.have.nested.property('cid.version', 0) - await ipfs.files.write(filePath, Buffer.from([5, 6]), { + await ipfs.files.write(filePath, Uint8Array.from([5, 6]), { create: true, cidVersion: 0 }) @@ -668,7 +668,7 @@ module.exports = (common, options) => { await expect(ipfs.files.stat(filePath)).to.eventually.have.nested.property('cid.version', 1) - const actualBytes = Buffer.concat(await all(ipfs.files.read(filePath))) + const actualBytes = uint8ArrayConcat(await all(ipfs.files.read(filePath))) expect(actualBytes).to.deep.equal(expectedBytes) }) @@ -685,23 +685,23 @@ module.exports = (common, options) => { await expect(ipfs.files.stat(directoryPath)).to.eventually.have.nested.property('cid.version', 0) - await ipfs.files.write(filePath, Buffer.from([5, 6, 7, 8, 9, 10, 11]), { + await ipfs.files.write(filePath, Uint8Array.from([5, 6, 7, 8, 9, 10, 11]), { create: true, cidVersion: 0 }) await expect(ipfs.files.stat(filePath)).to.eventually.have.nested.property('cid.version', 0) - await ipfs.files.write(filePath, Buffer.from([0, 1, 2, 3]), { + await ipfs.files.write(filePath, Uint8Array.from([0, 1, 2, 3]), { cidVersion: 1, offset: 1 }) await expect(ipfs.files.stat(filePath)).to.eventually.have.nested.property('cid.version', 1) - const actualBytes = Buffer.concat(await all(ipfs.files.read(filePath))) + const actualBytes = uint8ArrayConcat(await all(ipfs.files.read(filePath))) - expect(actualBytes).to.deep.equal(Buffer.from([5, 0, 1, 2, 3, 10, 11])) + expect(actualBytes).to.deep.equal(Uint8Array.from([5, 0, 1, 2, 3, 10, 11])) }) it('writes a file with a different hash function to the parent', async () => { @@ -709,7 +709,7 @@ module.exports = (common, options) => { const directoryPath = `/${directory}` const fileName = `file-${Math.random()}.txt` const filePath = `${directoryPath}/${fileName}` - const expectedBytes = Buffer.from([0, 1, 2, 3]) + const expectedBytes = Uint8Array.from([0, 1, 2, 3]) await ipfs.files.mkdir(directoryPath, { cidVersion: 0 @@ -728,7 +728,7 @@ module.exports = (common, options) => { return multihash.decode(hash).name === 'sha2-512' }) - const actualBytes = Buffer.concat(await all(ipfs.files.read(filePath))) + const actualBytes = uint8ArrayConcat(await all(ipfs.files.read(filePath))) expect(actualBytes).to.deep.equal(expectedBytes) }) diff --git a/packages/interface-ipfs-core/src/get.js b/packages/interface-ipfs-core/src/get.js index c2a7b1f201..8080897934 100644 --- a/packages/interface-ipfs-core/src/get.js +++ b/packages/interface-ipfs-core/src/get.js @@ -1,11 +1,12 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require('uint8arrays/to-string') +const uint8ArrayConcat = require('uint8arrays/concat') const { fixtures } = require('./utils') const CID = require('cids') const all = require('it-all') -const concat = require('it-concat') const drain = require('it-drain') const last = require('it-last') const { getDescribe, getIt, expect } = require('./utils/mocha') @@ -28,8 +29,8 @@ module.exports = (common, options) => { before(async () => { ipfs = (await common.spawn()).api - await all(importer([{ content: fixtures.smallFile.data }], ipfs.block)) - await all(importer([{ content: fixtures.bigFile.data }], ipfs.block)) + await drain(importer([{ content: fixtures.smallFile.data }], ipfs.block)) + await drain(importer([{ content: fixtures.bigFile.data }], ipfs.block)) }) after(() => common.clean()) @@ -44,20 +45,20 @@ module.exports = (common, options) => { const files = await all(ipfs.get(fixtures.smallFile.cid)) expect(files).to.be.length(1) expect(files[0].path).to.eql(fixtures.smallFile.cid) - expect((await concat(files[0].content)).toString()).to.contain('Plz add me!') + expect(uint8ArrayToString(uint8ArrayConcat(await all(files[0].content)))).to.contain('Plz add me!') }) - it('should get with a Buffer multihash', async () => { + it('should get with a Uint8Array multihash', async () => { const cidBuf = new CID(fixtures.smallFile.cid).multihash const files = await all(ipfs.get(cidBuf)) expect(files).to.be.length(1) expect(files[0].path).to.eql(fixtures.smallFile.cid) - expect((await concat(files[0].content)).toString()).to.contain('Plz add me!') + expect(uint8ArrayToString(uint8ArrayConcat(await all(files[0].content)))).to.contain('Plz add me!') }) it('should get a file added as CIDv0 with a CIDv1', async () => { - const input = Buffer.from(`TEST${Math.random()}`) + const input = uint8ArrayFromString(`TEST${Math.random()}`) const res = await all(importer([{ content: input }], ipfs.block)) @@ -67,11 +68,11 @@ module.exports = (common, options) => { const cidv1 = cidv0.toV1() const output = await all(ipfs.get(cidv1)) - expect((await concat(output[0].content)).slice()).to.eql(input) + expect(uint8ArrayConcat(await all(output[0].content))).to.eql(input) }) it('should get a file added as CIDv1 with a CIDv0', async () => { - const input = Buffer.from(`TEST${Math.random()}`) + const input = uint8ArrayFromString(`TEST${Math.random()}`) const res = await all(importer([{ content: input }], ipfs.block, { cidVersion: 1, rawLeaves: false })) @@ -81,13 +82,13 @@ module.exports = (common, options) => { const cidv0 = cidv1.toV0() const output = await all(ipfs.get(cidv0)) - expect((await concat(output[0].content)).slice()).to.eql(input) + expect(uint8ArrayConcat(await all(output[0].content))).to.eql(input) }) it('should get a BIG file', async () => { for await (const file of ipfs.get(fixtures.bigFile.cid)) { expect(file.path).to.equal(fixtures.bigFile.cid) - const content = await concat(file.content) + const content = uint8ArrayConcat(await all(file.content)) expect(content.length).to.eql(fixtures.bigFile.data.length) expect(content.slice()).to.eql(fixtures.bigFile.data) } @@ -120,7 +121,7 @@ module.exports = (common, options) => { let files = await all((async function * () { for await (let { path, content } of ipfs.get(fixtures.directory.cid)) { - content = content ? (await concat(content)).toString() : null + content = content ? uint8ArrayToString(uint8ArrayConcat(await all(content))) : null yield { path, content } } })()) @@ -168,7 +169,7 @@ module.exports = (common, options) => { const files = await all(ipfs.get(`/ipfs/${fileAdded.cid}/testfile.txt`)) expect(files).to.be.length(1) - expect((await concat(files[0].content)).toString()).to.contain('Plz add me!') + expect(uint8ArrayToString(uint8ArrayConcat(await all(files[0].content)))).to.contain('Plz add me!') }) it('should error on invalid key', async () => { diff --git a/packages/interface-ipfs-core/src/ls.js b/packages/interface-ipfs-core/src/ls.js index d1ba733631..e796fa3171 100644 --- a/packages/interface-ipfs-core/src/ls.js +++ b/packages/interface-ipfs-core/src/ls.js @@ -1,7 +1,6 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const { fixtures } = require('./utils') const { getDescribe, getIt, expect } = require('./utils/mocha') const all = require('it-all') @@ -113,8 +112,8 @@ module.exports = (common, options) => { const dir = randomName('DIR') const input = [ - { path: `${dir}/${randomName('F0')}`, content: Buffer.from(randomName('D0')) }, - { path: `${dir}/${randomName('F1')}`, content: Buffer.from(randomName('D1')) } + { path: `${dir}/${randomName('F0')}`, content: randomName('D0') }, + { path: `${dir}/${randomName('F1')}`, content: randomName('D1') } ] const res = await all(ipfs.addAll(input, { cidVersion: 0 })) @@ -136,8 +135,8 @@ module.exports = (common, options) => { const dir = randomName('DIR') const input = [ - { path: `${dir}/${randomName('F0')}`, content: Buffer.from(randomName('D0')) }, - { path: `${dir}/${randomName('F1')}`, content: Buffer.from(randomName('D1')) } + { path: `${dir}/${randomName('F0')}`, content: randomName('D0') }, + { path: `${dir}/${randomName('F1')}`, content: randomName('D1') } ] const res = await all(ipfs.addAll(input, { cidVersion: 1, rawLeaves: false })) @@ -167,8 +166,8 @@ module.exports = (common, options) => { const dir = randomName('DIR') const input = [ - { path: `${dir}/${randomName('F0')}`, content: Buffer.from(randomName('D0')) }, - { path: `${dir}/${randomName('F1')}`, content: Buffer.from(randomName('D1')) } + { path: `${dir}/${randomName('F0')}`, content: randomName('D0') }, + { path: `${dir}/${randomName('F1')}`, content: randomName('D1') } ] const res = await all(ipfs.addAll(input)) @@ -191,8 +190,8 @@ module.exports = (common, options) => { } const input = [ - { path: `${dir}/${randomName('F0')}`, content: Buffer.from(randomName('D0')), mode, mtime }, - { path: `${dir}/${randomName('F1')}`, content: Buffer.from(randomName('D1')), mode, mtime } + { path: `${dir}/${randomName('F0')}`, content: randomName('D0'), mode, mtime }, + { path: `${dir}/${randomName('F1')}`, content: randomName('D1'), mode, mtime } ] const res = await all(ipfs.addAll(input)) @@ -210,7 +209,7 @@ module.exports = (common, options) => { const subdir = randomName('F0') const subfile = randomName('F1') - const input = { path: `${dir}/${subdir}/${subfile}`, content: Buffer.from(randomName('D1')) } + const input = { path: `${dir}/${subdir}/${subfile}`, content: randomName('D1') } const res = await ipfs.add(input) const path = `${res.cid}/${subdir}` diff --git a/packages/interface-ipfs-core/src/miscellaneous/resolve.js b/packages/interface-ipfs-core/src/miscellaneous/resolve.js index a773529491..e70978766a 100644 --- a/packages/interface-ipfs-core/src/miscellaneous/resolve.js +++ b/packages/interface-ipfs-core/src/miscellaneous/resolve.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const isIpfs = require('is-ipfs') const loadFixture = require('aegir/fixtures') const { nanoid } = require('nanoid') @@ -45,7 +45,7 @@ module.exports = (common, options) => { }) it('should resolve an IPFS hash and return a base64url encoded CID in path', async () => { - const { cid } = await ipfs.add(Buffer.from('base64url encoded')) + const { cid } = await ipfs.add(uint8ArrayFromString('base64url encoded')) const path = await ipfs.resolve(`/ipfs/${cid}`, { cidBase: 'base64url' }) const [,, cidStr] = path.split('/') @@ -94,7 +94,7 @@ module.exports = (common, options) => { // webworkers are not dialable because webrtc is not available const node = (await common.spawn({ type: isWebWorker ? 'go' : undefined })).api await ipfs.swarm.connect(node.peerId.addresses[0]) - const { path } = await ipfs.add(Buffer.from('should resolve a record recursive === true')) + const { path } = await ipfs.add(uint8ArrayFromString('should resolve a record recursive === true')) const { id: keyId } = await ipfs.key.gen('key-name', { type: 'rsa', size: 2048 }) await ipfs.name.publish(path, { allowOffline: true }) diff --git a/packages/interface-ipfs-core/src/name-pubsub/cancel.js b/packages/interface-ipfs-core/src/name-pubsub/cancel.js index 24621403db..7b00a5cd6a 100644 --- a/packages/interface-ipfs-core/src/name-pubsub/cancel.js +++ b/packages/interface-ipfs-core/src/name-pubsub/cancel.js @@ -38,7 +38,7 @@ module.exports = (common, options) => { const res = await ipfs.name.pubsub.cancel(nodeId) expect(res).to.exist() expect(res).to.have.property('canceled') - expect(res.canceled).to.eql(false) + expect(res.canceled).to.be.false() }) it('should cancel a subscription correctly returning true', async function () { @@ -57,9 +57,9 @@ module.exports = (common, options) => { const cancel = await ipfs.name.pubsub.cancel(ipnsPath) const subs2 = await ipfs.name.pubsub.subs() - expect(subs1).to.be.an('array').that.does.include(ipnsPath) + expect(subs1).to.be.an('array').that.includes(ipnsPath) expect(cancel).to.have.property('canceled') - expect(cancel.canceled).to.eql(true) + expect(cancel.canceled).to.be.true() expect(subs2).to.be.an('array').that.does.not.include(ipnsPath) }) }) diff --git a/packages/interface-ipfs-core/src/name/publish.js b/packages/interface-ipfs-core/src/name/publish.js index c2af18be63..b83ee5c987 100644 --- a/packages/interface-ipfs-core/src/name/publish.js +++ b/packages/interface-ipfs-core/src/name/publish.js @@ -2,7 +2,7 @@ 'use strict' const { nanoid } = require('nanoid') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { fixture } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const last = require('it-last') @@ -49,7 +49,7 @@ module.exports = (common, options) => { }) it('should publish correctly with the lifetime option and resolve', async () => { - const { path } = await ipfs.add(Buffer.from('should publish correctly with the lifetime option and resolve')) + const { path } = await ipfs.add(uint8ArrayFromString('should publish correctly with the lifetime option and resolve')) await ipfs.name.publish(path, { allowOffline: true, resolve: false, lifetime: '2h' }) expect(await last(ipfs.name.resolve(`/ipns/${nodeId}`))).to.eq(`/ipfs/${path}`) }) diff --git a/packages/interface-ipfs-core/src/name/resolve.js b/packages/interface-ipfs-core/src/name/resolve.js index 8465bdc6bf..917eb50d6b 100644 --- a/packages/interface-ipfs-core/src/name/resolve.js +++ b/packages/interface-ipfs-core/src/name/resolve.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { getDescribe, getIt, expect } = require('../utils/mocha') const delay = require('delay') const CID = require('cids') @@ -30,7 +30,7 @@ module.exports = (common, options) => { it('should resolve a record default options', async function () { this.timeout(20 * 1000) - const { path } = await ipfs.add(Buffer.from('should resolve a record default options')) + const { path } = await ipfs.add(uint8ArrayFromString('should resolve a record default options')) const { id: keyId } = await ipfs.key.gen('key-name-default', { type: 'rsa', size: 2048 }) await ipfs.name.publish(path, { allowOffline: true }) @@ -42,7 +42,7 @@ module.exports = (common, options) => { it('should resolve a record from peerid as cidv1 in base32', async function () { this.timeout(20 * 1000) - const { path } = await ipfs.add(Buffer.from('should resolve a record from cidv1b32')) + const { path } = await ipfs.add(uint8ArrayFromString('should resolve a record from cidv1b32')) const { id: peerId } = await ipfs.id() await ipfs.name.publish(path, { allowOffline: true }) @@ -55,7 +55,7 @@ module.exports = (common, options) => { }) it('should resolve a record recursive === false', async () => { - const { path } = await ipfs.add(Buffer.from('should resolve a record recursive === false')) + const { path } = await ipfs.add(uint8ArrayFromString('should resolve a record recursive === false')) await ipfs.name.publish(path, { allowOffline: true }) expect(await last(ipfs.name.resolve(`/ipns/${nodeId}`, { recursive: false }))) .to.eq(`/ipfs/${path}`) @@ -64,7 +64,7 @@ module.exports = (common, options) => { it('should resolve a record recursive === true', async function () { this.timeout(20 * 1000) - const { path } = await ipfs.add(Buffer.from('should resolve a record recursive === true')) + const { path } = await ipfs.add(uint8ArrayFromString('should resolve a record recursive === true')) const { id: keyId } = await ipfs.key.gen('key-name', { type: 'rsa', size: 2048 }) await ipfs.name.publish(path, { allowOffline: true }) @@ -77,7 +77,7 @@ module.exports = (common, options) => { it('should resolve a record default options with remainder', async function () { this.timeout(20 * 1000) - const { path } = await ipfs.add(Buffer.from('should resolve a record default options with remainder')) + const { path } = await ipfs.add(uint8ArrayFromString('should resolve a record default options with remainder')) const { id: keyId } = await ipfs.key.gen('key-name-remainder-default', { type: 'rsa', size: 2048 }) await ipfs.name.publish(path, { allowOffline: true }) @@ -88,7 +88,7 @@ module.exports = (common, options) => { }) it('should resolve a record recursive === false with remainder', async () => { - const { path } = await ipfs.add(Buffer.from('should resolve a record recursive = false with remainder')) + const { path } = await ipfs.add(uint8ArrayFromString('should resolve a record recursive = false with remainder')) await ipfs.name.publish(path, { allowOffline: true }) expect(await last(ipfs.name.resolve(`/ipns/${nodeId}/remainder/file.txt`, { recursive: false }))) .to.eq(`/ipfs/${path}/remainder/file.txt`) @@ -97,7 +97,7 @@ module.exports = (common, options) => { it('should resolve a record recursive === true with remainder', async function () { this.timeout(20 * 1000) - const { path } = await ipfs.add(Buffer.from('should resolve a record recursive = true with remainder')) + const { path } = await ipfs.add(uint8ArrayFromString('should resolve a record recursive = true with remainder')) const { id: keyId } = await ipfs.key.gen('key-name-remainder', { type: 'rsa', size: 2048 }) await ipfs.name.publish(path, { allowOffline: true }) @@ -115,7 +115,7 @@ module.exports = (common, options) => { } // we add new data instead of re-using fixture to make sure lifetime handling doesn't break - const { path } = await ipfs.add(Buffer.from('should not get the entry if its validity time expired')) + const { path } = await ipfs.add(uint8ArrayFromString('should not get the entry if its validity time expired')) await ipfs.name.publish(path, publishOptions) await delay(500) // go only has 1 possible error https://github.com/ipfs/go-ipfs/blob/master/namesys/interface.go#L51 diff --git a/packages/interface-ipfs-core/src/object/data.js b/packages/interface-ipfs-core/src/object/data.js index a8038c49ce..fab08955c1 100644 --- a/packages/interface-ipfs-core/src/object/data.js +++ b/packages/interface-ipfs-core/src/object/data.js @@ -1,11 +1,11 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const { nanoid } = require('nanoid') const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') const CID = require('cids') +const uint8ArrayFromString = require('uint8arrays/from-string') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -35,7 +35,7 @@ module.exports = (common, options) => { it('should get data by multihash', async () => { const testObj = { - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [] } @@ -47,7 +47,7 @@ module.exports = (common, options) => { it('should get data by base58 encoded multihash string', async () => { const testObj = { - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [] } diff --git a/packages/interface-ipfs-core/src/object/get.js b/packages/interface-ipfs-core/src/object/get.js index 85a8246a3b..7ed9ef4df1 100644 --- a/packages/interface-ipfs-core/src/object/get.js +++ b/packages/interface-ipfs-core/src/object/get.js @@ -1,7 +1,6 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const dagPB = require('ipld-dag-pb') const DAGNode = dagPB.DAGNode const { nanoid } = require('nanoid') @@ -11,6 +10,7 @@ const randomBytes = require('iso-random-stream/src/random') const { asDAGLink } = require('./utils') const testTimeout = require('../utils/test-timeout') const CID = require('cids') +const uint8ArrayFromString = require('uint8arrays/from-string') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -40,7 +40,7 @@ module.exports = (common, options) => { it('should get object by multihash', async () => { const obj = { - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [] } @@ -51,7 +51,7 @@ module.exports = (common, options) => { // because js-ipfs-api can't infer if the // returned Data is Buffer or String if (typeof node2.Data === 'string') { - node2 = new DAGNode(Buffer.from(node2.Data), node2.Links, node2.size) + node2 = new DAGNode(uint8ArrayFromString(node2.Data), node2.Links, node2.size) } expect(node1.Data).to.eql(node2.Data) @@ -60,7 +60,7 @@ module.exports = (common, options) => { it('should get object by multihash string', async () => { const obj = { - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [] } @@ -71,7 +71,7 @@ module.exports = (common, options) => { // because js-ipfs-api can't infer if the // returned Data is Buffer or String if (typeof node2.Data === 'string') { - node2 = new DAGNode(Buffer.from(node2.Data), node2.Links, node2.size) + node2 = new DAGNode(uint8ArrayFromString(node2.Data), node2.Links, node2.size) } expect(node1.Data).to.deep.equal(node2.Data) @@ -79,8 +79,8 @@ module.exports = (common, options) => { }) it('should get object with links by multihash string', async () => { - const node1a = new DAGNode(Buffer.from('Some data 1')) - const node2 = new DAGNode(Buffer.from('Some data 2')) + const node1a = new DAGNode(uint8ArrayFromString('Some data 1')) + const node2 = new DAGNode(uint8ArrayFromString('Some data 2')) const link = await asDAGLink(node2, 'some-link') const node1b = new DAGNode(node1a.Data, node1a.Links.concat(link)) @@ -91,7 +91,7 @@ module.exports = (common, options) => { // because js-ipfs-api can't infer if the // returned Data is Buffer or String if (typeof node1c.Data === 'string') { - node1c = new DAGNode(Buffer.from(node1c.Data), node1c.Links, node1c.size) + node1c = new DAGNode(uint8ArrayFromString(node1c.Data), node1c.Links, node1c.size) } expect(node1a.Data).to.eql(node1c.Data) @@ -99,7 +99,7 @@ module.exports = (common, options) => { it('should get object by base58 encoded multihash', async () => { const obj = { - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [] } @@ -110,7 +110,7 @@ module.exports = (common, options) => { // because js-ipfs-api can't infer if the // returned Data is Buffer or String if (typeof node1b.Data === 'string') { - node1b = new DAGNode(Buffer.from(node1b.Data), node1b.Links, node1b.size) + node1b = new DAGNode(uint8ArrayFromString(node1b.Data), node1b.Links, node1b.size) } expect(node1a.Data).to.eql(node1b.Data) @@ -119,7 +119,7 @@ module.exports = (common, options) => { it('should get object by base58 encoded multihash string', async () => { const obj = { - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [] } @@ -130,7 +130,7 @@ module.exports = (common, options) => { // because js-ipfs-api can't infer if the // returned Data is Buffer or String if (typeof node1b.Data === 'string') { - node1b = new DAGNode(Buffer.from(node1b.Data), node1b.Links, node1b.size) + node1b = new DAGNode(uint8ArrayFromString(node1b.Data), node1b.Links, node1b.size) } expect(node1a.Data).to.eql(node1b.Data) diff --git a/packages/interface-ipfs-core/src/object/links.js b/packages/interface-ipfs-core/src/object/links.js index 30f52274a8..c394270920 100644 --- a/packages/interface-ipfs-core/src/object/links.js +++ b/packages/interface-ipfs-core/src/object/links.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const dagPB = require('ipld-dag-pb') const DAGNode = dagPB.DAGNode const { nanoid } = require('nanoid') @@ -38,7 +38,7 @@ module.exports = (common, options) => { it('should get empty links by multihash', async () => { const testObj = { - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [] } @@ -50,8 +50,8 @@ module.exports = (common, options) => { }) it('should get links by multihash', async () => { - const node1a = new DAGNode(Buffer.from('Some data 1')) - const node2 = new DAGNode(Buffer.from('Some data 2')) + const node1a = new DAGNode(uint8ArrayFromString('Some data 1')) + const node2 = new DAGNode(uint8ArrayFromString('Some data 2')) const link = await asDAGLink(node2, 'some-link') @@ -66,20 +66,20 @@ module.exports = (common, options) => { it('should get links by base58 encoded multihash', async () => { const testObj = { - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [] } const cid = await ipfs.object.put(testObj) const node = await ipfs.object.get(cid) - const links = await ipfs.object.links(cid.buffer, { enc: 'base58' }) + const links = await ipfs.object.links(cid.bytes, { enc: 'base58' }) expect(node.Links).to.deep.equal(links) }) it('should get links by base58 encoded multihash string', async () => { const testObj = { - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [] } @@ -93,10 +93,10 @@ module.exports = (common, options) => { it('should get links from CBOR object', async () => { const hashes = [] - const res1 = await ipfs.add(Buffer.from('test data')) + const res1 = await ipfs.add(uint8ArrayFromString('test data')) hashes.push(res1.cid) - const res2 = await ipfs.add(Buffer.from('more test data')) + const res2 = await ipfs.add(uint8ArrayFromString('more test data')) hashes.push(res2.cid) const obj = { diff --git a/packages/interface-ipfs-core/src/object/patch/add-link.js b/packages/interface-ipfs-core/src/object/patch/add-link.js index a480bda2f2..5e1d34f6b2 100644 --- a/packages/interface-ipfs-core/src/object/patch/add-link.js +++ b/packages/interface-ipfs-core/src/object/patch/add-link.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const dagPB = require('ipld-dag-pb') const DAGNode = dagPB.DAGNode const { getDescribe, getIt, expect } = require('../../utils/mocha') @@ -37,11 +37,11 @@ module.exports = (common, options) => { it('should add a link to an existing node', async () => { const obj = { - Data: Buffer.from('patch test object'), + Data: uint8ArrayFromString('patch test object'), Links: [] } // link to add - const node2 = new DAGNode(Buffer.from('some other node')) + const node2 = new DAGNode(uint8ArrayFromString('some other node')) // note: we need to put the linked obj, otherwise IPFS won't // timeout. Reason: it needs the node to get its size await ipfs.object.put(node2) @@ -61,7 +61,7 @@ module.exports = (common, options) => { /* TODO: revisit this assertions. // note: make sure we can link js plain objects - const content = Buffer.from(JSON.stringify({ + const content = uint8ArrayFromString(JSON.stringify({ title: 'serialized object' }, null, 0)) const result = await ipfs.add(content) diff --git a/packages/interface-ipfs-core/src/object/patch/append-data.js b/packages/interface-ipfs-core/src/object/patch/append-data.js index ccdb6dd98f..26bf929208 100644 --- a/packages/interface-ipfs-core/src/object/patch/append-data.js +++ b/packages/interface-ipfs-core/src/object/patch/append-data.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { getDescribe, getIt, expect } = require('../../utils/mocha') const testTimeout = require('../../utils/test-timeout') const CID = require('cids') @@ -27,19 +27,19 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when appending data to an object', () => { - return testTimeout(() => ipfs.object.patch.appendData(new CID('Qmd7qZS4T7xXtsNFdRoK1trfMs5zU94EpokQ9WFtxdPxsZ'), Buffer.from('derp'), { + return testTimeout(() => ipfs.object.patch.appendData(new CID('Qmd7qZS4T7xXtsNFdRoK1trfMs5zU94EpokQ9WFtxdPxsZ'), uint8ArrayFromString('derp'), { timeout: 1 })) }) it('should append data to an existing node', async () => { const obj = { - Data: Buffer.from('patch test object'), + Data: uint8ArrayFromString('patch test object'), Links: [] } const nodeCid = await ipfs.object.put(obj) - const patchedNodeCid = await ipfs.object.patch.appendData(nodeCid, Buffer.from('append')) + const patchedNodeCid = await ipfs.object.patch.appendData(nodeCid, uint8ArrayFromString('append')) expect(patchedNodeCid).to.not.deep.equal(nodeCid) }) diff --git a/packages/interface-ipfs-core/src/object/patch/rm-link.js b/packages/interface-ipfs-core/src/object/patch/rm-link.js index 2269266062..653a53c84b 100644 --- a/packages/interface-ipfs-core/src/object/patch/rm-link.js +++ b/packages/interface-ipfs-core/src/object/patch/rm-link.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { getDescribe, getIt, expect } = require('../../utils/mocha') const { asDAGLink } = require('../utils') const testTimeout = require('../../utils/test-timeout') @@ -35,12 +35,12 @@ module.exports = (common, options) => { it('should remove a link from an existing node', async () => { const obj1 = { - Data: Buffer.from('patch test object 1'), + Data: uint8ArrayFromString('patch test object 1'), Links: [] } const obj2 = { - Data: Buffer.from('patch test object 2'), + Data: uint8ArrayFromString('patch test object 2'), Links: [] } diff --git a/packages/interface-ipfs-core/src/object/patch/set-data.js b/packages/interface-ipfs-core/src/object/patch/set-data.js index 0002bf0363..7c80a67fd0 100644 --- a/packages/interface-ipfs-core/src/object/patch/set-data.js +++ b/packages/interface-ipfs-core/src/object/patch/set-data.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { getDescribe, getIt, expect } = require('../../utils/mocha') const testTimeout = require('../../utils/test-timeout') const CID = require('cids') @@ -27,17 +27,17 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when setting the data of an object', () => { - return testTimeout(() => ipfs.object.patch.setData(new CID('Qmd7qZS4T7xXtsNFdRoK1trfMs5zU94EpokQ9WFtxdPxsZ'), Buffer.from('derp'), { + return testTimeout(() => ipfs.object.patch.setData(new CID('Qmd7qZS4T7xXtsNFdRoK1trfMs5zU94EpokQ9WFtxdPxsZ'), uint8ArrayFromString('derp'), { timeout: 1 })) }) it('should set data for an existing node', async () => { const obj = { - Data: Buffer.from('patch test object'), + Data: uint8ArrayFromString('patch test object'), Links: [] } - const patchData = Buffer.from('set') + const patchData = uint8ArrayFromString('set') const nodeCid = await ipfs.object.put(obj) const patchedNodeCid = await ipfs.object.patch.setData(nodeCid, patchData) diff --git a/packages/interface-ipfs-core/src/object/put.js b/packages/interface-ipfs-core/src/object/put.js index 8c84afc8d4..5fbfa32c66 100644 --- a/packages/interface-ipfs-core/src/object/put.js +++ b/packages/interface-ipfs-core/src/object/put.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const dagPB = require('ipld-dag-pb') const DAGNode = dagPB.DAGNode const { nanoid } = require('nanoid') @@ -30,14 +30,14 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when putting an object', () => { - return testTimeout(() => ipfs.object.put({ Data: Buffer.from('derp') }, { + return testTimeout(() => ipfs.object.put({ Data: uint8ArrayFromString('derp') }, { timeout: 1 })) }) it('should put an object', async () => { const obj = { - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [] } @@ -51,7 +51,7 @@ module.exports = (common, options) => { it('should put a JSON encoded Buffer', async () => { const obj = { - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [] } @@ -60,7 +60,7 @@ module.exports = (common, options) => { Links: obj.Links } - const buf = Buffer.from(JSON.stringify(obj2)) + const buf = uint8ArrayFromString(JSON.stringify(obj2)) const cid = await ipfs.object.put(buf, { enc: 'json' }) @@ -70,7 +70,7 @@ module.exports = (common, options) => { }) it('should put a Protobuf encoded Buffer', async () => { - const node = new DAGNode(Buffer.from(nanoid())) + const node = new DAGNode(uint8ArrayFromString(nanoid())) const serialized = node.serialize() const cid = await ipfs.object.put(serialized, { enc: 'protobuf' }) @@ -80,7 +80,7 @@ module.exports = (common, options) => { }) it('should put a Buffer as data', async () => { - const data = Buffer.from(nanoid()) + const data = uint8ArrayFromString(nanoid()) const cid = await ipfs.object.put(data) const node = await ipfs.object.get(cid) @@ -90,7 +90,7 @@ module.exports = (common, options) => { }) it('should put a Protobuf DAGNode', async () => { - const dNode = new DAGNode(Buffer.from(nanoid())) + const dNode = new DAGNode(uint8ArrayFromString(nanoid())) const cid = await ipfs.object.put(dNode) const node = await ipfs.object.get(cid) @@ -103,8 +103,8 @@ module.exports = (common, options) => { }) it('should put a Protobuf DAGNode with a link', async () => { - const node1a = new DAGNode(Buffer.from(nanoid())) - const node2 = new DAGNode(Buffer.from(nanoid())) + const node1a = new DAGNode(uint8ArrayFromString(nanoid())) + const node2 = new DAGNode(uint8ArrayFromString(nanoid())) const link = await asDAGLink(node2, 'some-link') diff --git a/packages/interface-ipfs-core/src/object/stat.js b/packages/interface-ipfs-core/src/object/stat.js index c7d18a87e1..cad848accd 100644 --- a/packages/interface-ipfs-core/src/object/stat.js +++ b/packages/interface-ipfs-core/src/object/stat.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const dagPB = require('ipld-dag-pb') const DAGNode = dagPB.DAGNode const { getDescribe, getIt, expect } = require('../utils/mocha') @@ -37,7 +37,7 @@ module.exports = (common, options) => { it('should get stats by multihash', async () => { const testObj = { - Data: Buffer.from('get test object'), + Data: uint8ArrayFromString('get test object'), Links: [] } @@ -55,8 +55,8 @@ module.exports = (common, options) => { }) it('should get stats for object with links by multihash', async () => { - const node1a = new DAGNode(Buffer.from('Some data 1')) - const node2 = new DAGNode(Buffer.from('Some data 2')) + const node1a = new DAGNode(uint8ArrayFromString('Some data 1')) + const node2 = new DAGNode(uint8ArrayFromString('Some data 2')) const link = await asDAGLink(node2, 'some-link') @@ -77,13 +77,13 @@ module.exports = (common, options) => { it('should get stats by base58 encoded multihash', async () => { const testObj = { - Data: Buffer.from('get test object'), + Data: uint8ArrayFromString('get test object'), Links: [] } const cid = await ipfs.object.put(testObj) - const stats = await ipfs.object.stat(cid.buffer) + const stats = await ipfs.object.stat(cid.bytes) const expected = { Hash: 'QmNggDXca24S6cMPEYHZjeuc4QRmofkRrAEqVL3Ms2sdJZ', NumLinks: 0, @@ -97,7 +97,7 @@ module.exports = (common, options) => { it('should get stats by base58 encoded multihash string', async () => { const testObj = { - Data: Buffer.from('get test object'), + Data: uint8ArrayFromString('get test object'), Links: [] } diff --git a/packages/interface-ipfs-core/src/pin/add.js b/packages/interface-ipfs-core/src/pin/add.js index 6010a24151..32f2d46cdf 100644 --- a/packages/interface-ipfs-core/src/pin/add.js +++ b/packages/interface-ipfs-core/src/pin/add.js @@ -1,6 +1,7 @@ /* eslint-env mocha */ 'use strict' +const uint8ArrayFromString = require('uint8arrays/from-string') const { fixtures, clearPins, expectPinned, expectNotPinned, pinTypes } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const all = require('it-all') @@ -138,7 +139,7 @@ module.exports = (common, options) => { }) it('should pin raw', async () => { - const cid = await ipfs.dag.put(Buffer.alloc(0), { + const cid = await ipfs.dag.put(new Uint8Array(0), { format: 'raw', hashAlg: 'sha2-256' }) @@ -154,7 +155,7 @@ module.exports = (common, options) => { }) it('should pin dag-cbor with dag-pb child', async () => { - const child = await ipfs.dag.put(new DAGNode(Buffer.from(`${Math.random()}`)), { + const child = await ipfs.dag.put(new DAGNode(uint8ArrayFromString(`${Math.random()}`)), { format: 'dag-pb', hashAlg: 'sha2-256' }) diff --git a/packages/interface-ipfs-core/src/pubsub/publish.js b/packages/interface-ipfs-core/src/pubsub/publish.js index 6145c0961d..27e048eb97 100644 --- a/packages/interface-ipfs-core/src/pubsub/publish.js +++ b/packages/interface-ipfs-core/src/pubsub/publish.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { nanoid } = require('nanoid') const { getTopic } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') @@ -45,7 +45,7 @@ module.exports = (common, options) => { it('should publish message from buffer', () => { const topic = getTopic() - return ipfs.pubsub.publish(topic, Buffer.from(nanoid())) + return ipfs.pubsub.publish(topic, uint8ArrayFromString(nanoid())) }) it('should publish 10 times within time limit', async () => { @@ -53,7 +53,7 @@ module.exports = (common, options) => { const topic = getTopic() for (let i = 0; i < count; i++) { - await ipfs.pubsub.publish(topic, Buffer.from(nanoid())) + await ipfs.pubsub.publish(topic, uint8ArrayFromString(nanoid())) } }) }) diff --git a/packages/interface-ipfs-core/src/pubsub/subscribe.js b/packages/interface-ipfs-core/src/pubsub/subscribe.js index f486cc9120..01a8d7bb4a 100644 --- a/packages/interface-ipfs-core/src/pubsub/subscribe.js +++ b/packages/interface-ipfs-core/src/pubsub/subscribe.js @@ -1,7 +1,8 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require('uint8arrays/to-string') const { nanoid } = require('nanoid') const pushable = require('it-pushable') const all = require('it-all') @@ -63,12 +64,12 @@ module.exports = (common, options) => { msgStream.end() }) - await ipfs1.pubsub.publish(topic, Buffer.from('hi')) + await ipfs1.pubsub.publish(topic, uint8ArrayFromString('hi')) for await (const msg of msgStream) { - expect(msg.data.toString()).to.equal('hi') + expect(uint8ArrayToString(msg.data)).to.equal('hi') expect(msg).to.have.property('seqno') - expect(Buffer.isBuffer(msg.seqno)).to.eql(true) + expect(msg.seqno).to.be.an.instanceof(Uint8Array) expect(msg.topicIDs[0]).to.eq(topic) expect(msg).to.have.property('from', ipfs1.peerId.id) break @@ -83,12 +84,12 @@ module.exports = (common, options) => { msgStream.end() }, {}) - await ipfs1.pubsub.publish(topic, Buffer.from('hi')) + await ipfs1.pubsub.publish(topic, uint8ArrayFromString('hi')) for await (const msg of msgStream) { - expect(msg.data.toString()).to.equal('hi') + expect(uint8ArrayToString(msg.data)).to.equal('hi') expect(msg).to.have.property('seqno') - expect(Buffer.isBuffer(msg.seqno)).to.eql(true) + expect(msg.seqno).to.be.an.instanceof(Uint8Array) expect(msg.topicIDs[0]).to.eq(topic) expect(msg).to.have.property('from', ipfs1.peerId.id) } @@ -112,13 +113,13 @@ module.exports = (common, options) => { ipfs1.pubsub.subscribe(topic, handler2) ]) - await ipfs1.pubsub.publish(topic, Buffer.from('hello')) + await ipfs1.pubsub.publish(topic, uint8ArrayFromString('hello')) const [handler1Msg] = await all(msgStream1) - expect(handler1Msg.data.toString()).to.eql('hello') + expect(uint8ArrayToString(handler1Msg.data)).to.eql('hello') const [handler2Msg] = await all(msgStream2) - expect(handler2Msg.data.toString()).to.eql('hello') + expect(uint8ArrayToString(handler2Msg.data)).to.eql('hello') await ipfs1.pubsub.unsubscribe(topic, handler1) await delay(100) @@ -141,10 +142,10 @@ module.exports = (common, options) => { msgStream.end() }, { discover: true }) - await ipfs1.pubsub.publish(topic, Buffer.from('hi')) + await ipfs1.pubsub.publish(topic, uint8ArrayFromString('hi')) for await (const msg of msgStream) { - expect(msg.data.toString()).to.eql('hi') + expect(uint8ArrayToString(msg.data)).to.eql('hi') } }) }) @@ -212,14 +213,14 @@ module.exports = (common, options) => { ]) await waitForPeers(ipfs2, topic, [ipfs1.peerId.id], 30000) - await ipfs2.pubsub.publish(topic, Buffer.from(expectedString)) + await ipfs2.pubsub.publish(topic, uint8ArrayFromString(expectedString)) const [sub1Msg] = await all(msgStream1) - expect(sub1Msg.data.toString()).to.be.eql(expectedString) + expect(uint8ArrayToString(sub1Msg.data)).to.be.eql(expectedString) expect(sub1Msg.from).to.eql(ipfs2.peerId.id) const [sub2Msg] = await all(msgStream2) - expect(sub2Msg.data.toString()).to.be.eql(expectedString) + expect(uint8ArrayToString(sub2Msg.data)).to.be.eql(expectedString) expect(sub2Msg.from).to.eql(ipfs2.peerId.id) abort1.abort() abort2.abort() @@ -247,20 +248,20 @@ module.exports = (common, options) => { await waitForPeers(ipfs2, topic, [ipfs1.peerId.id], 30000) await delay(5000) // gossipsub need this delay https://github.com/libp2p/go-libp2p-pubsub/issues/331 - await ipfs2.pubsub.publish(topic, Buffer.from(expectedString)) + await ipfs2.pubsub.publish(topic, uint8ArrayFromString(expectedString)) const [sub1Msg] = await all(msgStream1) - expect(sub1Msg.data.toString()).to.be.eql(expectedString) + expect(uint8ArrayToString(sub1Msg.data)).to.be.eql(expectedString) expect(sub1Msg.from).to.eql(ipfs2.peerId.id) const [sub2Msg] = await all(msgStream2) - expect(sub2Msg.data.toString()).to.be.eql(expectedString) + expect(uint8ArrayToString(sub2Msg.data)).to.be.eql(expectedString) expect(sub2Msg.from).to.eql(ipfs2.peerId.id) }) it('should round trip a non-utf8 binary buffer', async () => { const expectedHex = 'a36161636179656162830103056164a16466666666f4' - const buffer = Buffer.from(expectedHex, 'hex') + const buffer = uint8ArrayFromString(expectedHex, 'base16') const msgStream1 = pushable() const msgStream2 = pushable() @@ -284,11 +285,11 @@ module.exports = (common, options) => { await ipfs2.pubsub.publish(topic, buffer) const [sub1Msg] = await all(msgStream1) - expect(sub1Msg.data.toString('hex')).to.be.eql(expectedHex) + expect(uint8ArrayToString(sub1Msg.data, 'base16')).to.be.eql(expectedHex) expect(sub1Msg.from).to.eql(ipfs2.peerId.id) const [sub2Msg] = await all(msgStream2) - expect(sub2Msg.data.toString('hex')).to.be.eql(expectedHex) + expect(uint8ArrayToString(sub2Msg.data, 'base16')).to.be.eql(expectedHex) expect(sub2Msg.from).to.eql(ipfs2.peerId.id) }) @@ -321,17 +322,17 @@ module.exports = (common, options) => { await delay(5000) // gossipsub need this delay https://github.com/libp2p/go-libp2p-pubsub/issues/331 for (let i = 0; i < outbox.length; i++) { - await ipfs2.pubsub.publish(topic, Buffer.from(outbox[i])) + await ipfs2.pubsub.publish(topic, uint8ArrayFromString(outbox[i])) } const sub1Msgs = await all(msgStream1) sub1Msgs.forEach(msg => expect(msg.from).to.eql(ipfs2.peerId.id)) - const inbox1 = sub1Msgs.map(msg => msg.data.toString()) + const inbox1 = sub1Msgs.map(msg => uint8ArrayToString(msg.data)) expect(inbox1.sort()).to.eql(outbox.sort()) const sub2Msgs = await all(msgStream2) sub2Msgs.forEach(msg => expect(msg.from).to.eql(ipfs2.peerId.id)) - const inbox2 = sub2Msgs.map(msg => msg.data.toString()) + const inbox2 = sub2Msgs.map(msg => uint8ArrayToString(msg.data)) expect(inbox2.sort()).to.eql(outbox.sort()) }) @@ -359,7 +360,7 @@ module.exports = (common, options) => { const startTime = new Date().getTime() for (let i = 0; i < count; i++) { - const msgData = Buffer.from(msgBase + i) + const msgData = uint8ArrayFromString(msgBase + i) await ipfs2.pubsub.publish(topic, msgData) } @@ -372,7 +373,7 @@ module.exports = (common, options) => { msgs.forEach(msg => { expect(msg.from).to.eql(ipfs2.peerId.id) - expect(msg.data.toString().startsWith(msgBase)).to.be.true() + expect(uint8ArrayToString(msg.data).startsWith(msgBase)).to.be.true() }) }) }) diff --git a/packages/interface-ipfs-core/src/refs-local.js b/packages/interface-ipfs-core/src/refs-local.js index f15aa5f6d2..9a46f0266d 100644 --- a/packages/interface-ipfs-core/src/refs-local.js +++ b/packages/interface-ipfs-core/src/refs-local.js @@ -8,6 +8,7 @@ const importer = require('ipfs-unixfs-importer') const drain = require('it-drain') const testTimeout = require('./utils/test-timeout') const CID = require('cids') +const uint8ArrayEquals = require('uint8arrays/equals') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -58,7 +59,7 @@ module.exports = (common, options) => { cids.find(cid => { const multihash = new CID(cid).multihash - return imported[0].cid.multihash.equals(multihash) + return uint8ArrayEquals(imported[0].cid.multihash, multihash) }) ).to.be.ok() @@ -66,7 +67,7 @@ module.exports = (common, options) => { cids.find(cid => { const multihash = new CID(cid).multihash - return imported[1].cid.multihash.equals(multihash) + return uint8ArrayEquals(imported[1].cid.multihash, multihash) }) ).to.be.ok() }) diff --git a/packages/interface-ipfs-core/src/refs.js b/packages/interface-ipfs-core/src/refs.js index c813415496..c9fda5309d 100644 --- a/packages/interface-ipfs-core/src/refs.js +++ b/packages/interface-ipfs-core/src/refs.js @@ -1,7 +1,6 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const { getDescribe, getIt, expect } = require('./utils/mocha') const loadFixture = require('aegir/fixtures') const CID = require('cids') @@ -357,7 +356,7 @@ function loadDagContent (ipfs, node) { } async function loadContent (ipfs, store, node) { - if (Buffer.isBuffer(node)) { + if (node instanceof Uint8Array) { return store.putData(node) } diff --git a/packages/interface-ipfs-core/src/repo/gc.js b/packages/interface-ipfs-core/src/repo/gc.js index bc79697b0b..6ab8a46c96 100644 --- a/packages/interface-ipfs-core/src/repo/gc.js +++ b/packages/interface-ipfs-core/src/repo/gc.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { getDescribe, getIt, expect } = require('../utils/mocha') const { DAGNode } = require('ipld-dag-pb') const all = require('it-all') @@ -34,7 +34,7 @@ module.exports = (common, options) => { }) it('should run garbage collection', async () => { - const res = await ipfs.add(Buffer.from('apples')) + const res = await ipfs.add(uint8ArrayFromString('apples')) const pinset = await all(ipfs.pin.ls()) expect(pinset.map(obj => obj.cid.toString())).includes(res.cid.toString()) @@ -53,7 +53,7 @@ module.exports = (common, options) => { // Add some data. Note: this will implicitly pin the data, which causes // some blocks to be added for the data itself and for the pinning // information that refers to the blocks - const addRes = await ipfs.add(Buffer.from('apples')) + const addRes = await ipfs.add(uint8ArrayFromString('apples')) const cid = addRes.cid // Get the list of local blocks after the add, should be bigger than @@ -86,7 +86,7 @@ module.exports = (common, options) => { const refsBeforeAdd = await all(ipfs.refs.local()) // Add a file to MFS - await ipfs.files.write('/test', Buffer.from('oranges'), { create: true }) + await ipfs.files.write('/test', uint8ArrayFromString('oranges'), { create: true }) const stats = await ipfs.files.stat('/test') expect(stats.type).to.equal('file') @@ -120,7 +120,7 @@ module.exports = (common, options) => { const refsBeforeAdd = await all(ipfs.refs.local()) // Add a file to MFS - await ipfs.files.write('/test', Buffer.from('peaches'), { create: true }) + await ipfs.files.write('/test', uint8ArrayFromString('peaches'), { create: true }) const stats = await ipfs.files.stat('/test') expect(stats.type).to.equal('file') const mfsFileCid = stats.cid @@ -175,14 +175,14 @@ module.exports = (common, options) => { const refsBeforeAdd = await all(ipfs.refs.local()) // Add some data - const addRes = await ipfs.add(Buffer.from('pears')) + const addRes = await ipfs.add(uint8ArrayFromString('pears')) const dataCid = addRes.cid // Unpin the data await ipfs.pin.rm(dataCid) // Create a link to the data from an object - const obj = await new DAGNode(Buffer.from('fruit'), [{ + const obj = await new DAGNode(uint8ArrayFromString('fruit'), [{ Name: 'p', Hash: dataCid, Tsize: addRes.size diff --git a/packages/interface-ipfs-core/src/utils/create-sharded-directory.js b/packages/interface-ipfs-core/src/utils/create-sharded-directory.js index 26c9d5f0ab..6009a0e3a9 100644 --- a/packages/interface-ipfs-core/src/utils/create-sharded-directory.js +++ b/packages/interface-ipfs-core/src/utils/create-sharded-directory.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const { expect } = require('./mocha') const isShardAtPath = require('./is-shard-at-path') const last = require('it-last') @@ -12,7 +11,7 @@ module.exports = async (ipfs, files = 1001) => { for (let i = 0; i < files; i++) { yield { path: `${dirPath}/file-${i}`, - content: Buffer.from([0, 1, 2, 3, 4, 5, i]) + content: Uint8Array.from([0, 1, 2, 3, 4, 5, i]) } } }())) diff --git a/packages/interface-ipfs-core/src/utils/create-two-shards.js b/packages/interface-ipfs-core/src/utils/create-two-shards.js index 73728e9755..cbacf2b4da 100644 --- a/packages/interface-ipfs-core/src/utils/create-two-shards.js +++ b/packages/interface-ipfs-core/src/utils/create-two-shards.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const { expect } = require('./mocha') const isShardAtPath = require('./is-shard-at-path') const last = require('it-last') @@ -9,7 +8,7 @@ const createTwoShards = async (ipfs, fileCount) => { const dirPath = `/sharded-dir-${Math.random()}` const files = new Array(fileCount).fill(0).map((_, index) => ({ path: `${dirPath}/file-${index}`, - content: Buffer.from([0, 1, 2, 3, 4, index]) + content: Uint8Array.from([0, 1, 2, 3, 4, index]) })) files[files.length - 1].path = `${dirPath}/file-${fileCount - 1}` diff --git a/packages/interface-ipfs-core/src/utils/traverse-leaf-nodes.js b/packages/interface-ipfs-core/src/utils/traverse-leaf-nodes.js index b5cf025a6e..7fab114fb4 100644 --- a/packages/interface-ipfs-core/src/utils/traverse-leaf-nodes.js +++ b/packages/interface-ipfs-core/src/utils/traverse-leaf-nodes.js @@ -1,11 +1,10 @@ 'use strict' -const { Buffer } = require('buffer') module.exports = function traverseLeafNodes (ipfs, cid) { async function * traverse (cid) { const { value: node } = await ipfs.dag.get(cid) - if (Buffer.isBuffer(node) || !node.Links.length) { + if (node instanceof Uint8Array || !node.Links.length) { yield { node, cid diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index dc6211cb7a..496ebab592 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -30,19 +30,15 @@ "dependencies": { "blob-to-it": "0.0.1", "browser-readablestream-to-it": "0.0.1", - "buffer": "^5.6.0", "cids": "^1.0.0", "err-code": "^2.0.0", - "ipfs-utils": "^2.2.2", + "ipfs-utils": "ipfs/js-ipfs-utils#revert-54-xhr", "it-all": "^1.0.1", "it-map": "^1.0.0", "it-peekable": "0.0.1" }, "devDependencies": { - "aegir": "^25.0.0", - "chai": "^4.2.0", - "chai-as-promised": "^7.1.1", - "delay": "^4.3.0", - "dirty-chai": "^2.0.1" + "aegir": "^26.0.0", + "delay": "^4.3.0" } } diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js index c55f938e0f..6144717350 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js @@ -1,7 +1,7 @@ 'use strict' const errCode = require('err-code') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const browserStreamToIt = require('browser-readablestream-to-it') const blobToIt = require('blob-to-it') const itPeekable = require('it-peekable') @@ -60,7 +60,7 @@ async function * toAsyncIterable (input) { } function toBuffer (chunk) { - return isBytes(chunk) ? chunk : Buffer.from(chunk) + return isBytes(chunk) ? chunk : uint8ArrayFromString(chunk) } module.exports = toAsyncIterable diff --git a/packages/ipfs-core-utils/src/to-cid-and-path.js b/packages/ipfs-core-utils/src/to-cid-and-path.js index ab15475850..01d09791a3 100644 --- a/packages/ipfs-core-utils/src/to-cid-and-path.js +++ b/packages/ipfs-core-utils/src/to-cid-and-path.js @@ -6,7 +6,7 @@ const errCode = require('err-code') const IPFS_PREFIX = '/ipfs/' const toCidAndPath = (string) => { - if (Buffer.isBuffer(string)) { + if (string instanceof Uint8Array) { try { string = new CID(string) } catch (err) { diff --git a/packages/ipfs-core-utils/test/files/format-mode.spec.js b/packages/ipfs-core-utils/test/files/format-mode.spec.js index 6f5ae6196a..675fdc39d1 100644 --- a/packages/ipfs-core-utils/test/files/format-mode.spec.js +++ b/packages/ipfs-core-utils/test/files/format-mode.spec.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ -const { expect } = require('../utils/chai') +const { expect } = require('aegir/utils/chai') const formatMode = require('../../src/files/format-mode') describe('format-mode', function () { diff --git a/packages/ipfs-core-utils/test/files/format-mtime.spec.js b/packages/ipfs-core-utils/test/files/format-mtime.spec.js index 0bdb6ab532..f6b026600b 100644 --- a/packages/ipfs-core-utils/test/files/format-mtime.spec.js +++ b/packages/ipfs-core-utils/test/files/format-mtime.spec.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ -const { expect } = require('../utils/chai') +const { expect } = require('aegir/utils/chai') const formatMtime = require('../../src/files/format-mtime') describe('format-mtime', function () { diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index 7e206c2bd3..383f327c99 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -1,9 +1,10 @@ 'use strict' /* eslint-env mocha */ -const { expect } = require('../utils/chai') + +const { expect } = require('aegir/utils/chai') const blobToIt = require('blob-to-it') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const all = require('it-all') const { Blob, ReadableStream } = require('ipfs-utils/src/globalthis') const { isBrowser, isWebWorker } = require('ipfs-utils/src/env') @@ -16,7 +17,7 @@ if (isBrowser || isWebWorker) { const STRING = () => 'hello world' const NEWSTRING = () => new String('hello world') // eslint-disable-line no-new-wrappers -const BUFFER = () => Buffer.from(STRING()) +const BUFFER = () => uint8ArrayFromString(STRING()) const ARRAY = () => Array.from(BUFFER()) const TYPEDARRAY = () => Uint8Array.from(ARRAY()) let BLOB diff --git a/packages/ipfs-core-utils/test/pins/normalise-input.spec.js b/packages/ipfs-core-utils/test/pins/normalise-input.spec.js index fa2857ce0b..04e05d605d 100644 --- a/packages/ipfs-core-utils/test/pins/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/pins/normalise-input.spec.js @@ -1,15 +1,12 @@ 'use strict' /* eslint-env mocha */ -const chai = require('chai') -const dirtyChai = require('dirty-chai') + +const { expect } = require('aegir/utils/chai') const normalise = require('../../src/pins/normalise-input') const all = require('it-all') const CID = require('cids') -chai.use(dirtyChai) -const expect = chai.expect - const STRING = () => '/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn/path/to/file.txt' const PLAIN_CID = () => new CID('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') const OBJECT_CID = () => ({ cid: new CID('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'), recursive: true, metadata: { key: 'hello world' } }) diff --git a/packages/ipfs-core-utils/test/utils/chai.js b/packages/ipfs-core-utils/test/utils/chai.js deleted file mode 100644 index 2b87a01f91..0000000000 --- a/packages/ipfs-core-utils/test/utils/chai.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict' - -const chai = require('chai') - -chai.use(require('dirty-chai')) -chai.use(require('chai-as-promised')) - -module.exports = { - expect: chai.expect -} diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index ac082a5326..418f4f4313 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -43,12 +43,11 @@ "abort-controller": "^3.0.0", "any-signal": "^1.1.0", "bignumber.js": "^9.0.0", - "buffer": "^5.6.0", "cids": "^1.0.0", "debug": "^4.1.0", "form-data": "^3.0.0", "ipfs-core-utils": "^0.3.1", - "ipfs-utils": "^2.2.2", + "ipfs-utils": "ipfs/js-ipfs-utils#revert-54-xhr", "ipld-block": "^0.10.0", "ipld-dag-cbor": "^0.17.0", "ipld-dag-pb": "^0.20.0", @@ -60,21 +59,22 @@ "it-to-stream": "^0.1.1", "merge-options": "^2.0.0", "multiaddr": "^8.0.0", - "multiaddr-to-uri": "^5.1.0", + "multiaddr-to-uri": "^6.0.0", "multibase": "^3.0.0", "multicodec": "^2.0.0", "multihashes": "^3.0.1", "nanoid": "^3.0.2", "node-fetch": "^2.6.0", "parse-duration": "^0.4.4", - "stream-to-it": "^0.2.1" + "stream-to-it": "^0.2.1", + "uint8arrays": "^1.1.0" }, "devDependencies": { - "aegir": "^25.0.0", + "aegir": "^26.0.0", "cross-env": "^7.0.0", "go-ipfs": "^0.6.0", "interface-ipfs-core": "^0.139.0", - "ipfsd-ctl": "^5.0.0", + "ipfsd-ctl": "^6.0.0", "it-all": "^1.0.1", "it-concat": "^1.0.0", "it-pipe": "^1.1.0", diff --git a/packages/ipfs-http-client/src/block/get.js b/packages/ipfs-http-client/src/block/get.js index 5564aef09a..f39aa6d96b 100644 --- a/packages/ipfs-http-client/src/block/get.js +++ b/packages/ipfs-http-client/src/block/get.js @@ -2,7 +2,6 @@ const Block = require('ipld-block') const CID = require('cids') -const { Buffer } = require('buffer') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') @@ -20,6 +19,6 @@ module.exports = configure(api => { headers: options.headers }) - return new Block(Buffer.from(await res.arrayBuffer()), cid) + return new Block(new Uint8Array(await res.arrayBuffer()), cid) } }) diff --git a/packages/ipfs-http-client/src/config/replace.js b/packages/ipfs-http-client/src/config/replace.js index 57fce31db6..63ced0e20a 100644 --- a/packages/ipfs-http-client/src/config/replace.js +++ b/packages/ipfs-http-client/src/config/replace.js @@ -1,6 +1,6 @@ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const multipartRequest = require('../lib/multipart-request') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') @@ -18,7 +18,7 @@ module.exports = configure(api => { signal, searchParams: toUrlSearchParams(options), ...( - await multipartRequest(Buffer.from(JSON.stringify(config)), controller, options.headers) + await multipartRequest(uint8ArrayFromString(JSON.stringify(config)), controller, options.headers) ) }) diff --git a/packages/ipfs-http-client/src/dag/get.js b/packages/ipfs-http-client/src/dag/get.js index 426a59a34a..1094b97377 100644 --- a/packages/ipfs-http-client/src/dag/get.js +++ b/packages/ipfs-http-client/src/dag/get.js @@ -18,16 +18,16 @@ module.exports = configure((api, options) => { return async (cid, options = {}) => { const resolved = await dagResolve(cid, options) const block = await getBlock(resolved.cid, options) - const dagResolver = resolvers[block.cid.codec] + const dagResolver = resolvers[resolved.cid.codec] if (!dagResolver) { throw Object.assign( - new Error(`Missing IPLD format "${block.cid.codec}"`), - { missingMulticodec: cid.codec } + new Error(`Missing IPLD format "${resolved.cid.codec}"`), + { missingMulticodec: resolved.cid.codec } ) } - if (block.cid.codec === 'raw' && !resolved.remPath) { + if (resolved.cid.codec === 'raw' && !resolved.remPath) { resolved.remainderPath = '/' } diff --git a/packages/ipfs-http-client/src/dht/find-peer.js b/packages/ipfs-http-client/src/dht/find-peer.js index 43bd9ff35b..40985143f3 100644 --- a/packages/ipfs-http-client/src/dht/find-peer.js +++ b/packages/ipfs-http-client/src/dht/find-peer.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const multiaddr = require('multiaddr') const configure = require('../lib/configure') @@ -13,7 +12,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(peerId) ? new CID(peerId) : peerId}`, + arg: `${peerId instanceof Uint8Array ? new CID(peerId) : peerId}`, ...options }), headers: options.headers diff --git a/packages/ipfs-http-client/src/dht/get.js b/packages/ipfs-http-client/src/dht/get.js index f1299e2097..30ec158f97 100644 --- a/packages/ipfs-http-client/src/dht/get.js +++ b/packages/ipfs-http-client/src/dht/get.js @@ -1,9 +1,10 @@ 'use strict' -const { Buffer } = require('buffer') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') const { Value } = require('./response-types') +const uint8ArrayToString = require('uint8arrays/to-string') +const uint8ArrayFromString = require('uint8arrays/from-string') module.exports = configure(api => { return async function get (key, options = {}) { @@ -11,7 +12,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: Buffer.isBuffer(key) ? key.toString() : key, + arg: key instanceof Uint8Array ? uint8ArrayToString(key) : key, ...options }), headers: options.headers @@ -19,7 +20,7 @@ module.exports = configure(api => { for await (const message of res.ndjson()) { if (message.Type === Value) { - return Buffer.from(message.Extra, 'base64') + return uint8ArrayFromString(message.Extra, 'base64pad') } } diff --git a/packages/ipfs-http-client/src/files/read.js b/packages/ipfs-http-client/src/files/read.js index a85e33a1d4..fb1fc048a1 100644 --- a/packages/ipfs-http-client/src/files/read.js +++ b/packages/ipfs-http-client/src/files/read.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const toIterable = require('stream-to-it/source') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') @@ -18,8 +17,6 @@ module.exports = configure(api => { headers: options.headers }) - for await (const chunk of toIterable(res.body)) { - yield Buffer.from(chunk) - } + yield * toIterable(res.body) } }) diff --git a/packages/ipfs-http-client/src/get.js b/packages/ipfs-http-client/src/get.js index 43bcd18e01..387d5fd797 100644 --- a/packages/ipfs-http-client/src/get.js +++ b/packages/ipfs-http-client/src/get.js @@ -1,10 +1,10 @@ 'use strict' const Tar = require('it-tar') -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('./lib/configure') const toUrlSearchParams = require('./lib/to-url-search-params') +const map = require('it-map') module.exports = configure(api => { return async function * get (path, options = {}) { @@ -12,7 +12,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(path) ? new CID(path) : path}`, + arg: `${path instanceof Uint8Array ? new CID(path) : path}`, ...options }), headers: options.headers @@ -28,7 +28,7 @@ module.exports = configure(api => { } else { yield { path: header.name, - content: body + content: map(body, (chunk) => chunk.slice()) // convert bl to Buffer/Uint8Array } } } diff --git a/packages/ipfs-http-client/src/index.js b/packages/ipfs-http-client/src/index.js index aeb82ee703..3d0eeeedd0 100644 --- a/packages/ipfs-http-client/src/index.js +++ b/packages/ipfs-http-client/src/index.js @@ -1,6 +1,6 @@ 'use strict' /* eslint-env browser */ -const { Buffer } = require('buffer') + const CID = require('cids') const multiaddr = require('multiaddr') const multibase = require('multibase') @@ -56,6 +56,6 @@ function ipfsClient (options = {}) { } } -Object.assign(ipfsClient, { Buffer, CID, multiaddr, multibase, multicodec, multihash, globSource, urlSource }) +Object.assign(ipfsClient, { CID, multiaddr, multibase, multicodec, multihash, globSource, urlSource }) module.exports = ipfsClient diff --git a/packages/ipfs-http-client/src/ls.js b/packages/ipfs-http-client/src/ls.js index b448a0c2ee..ff9cdbb571 100644 --- a/packages/ipfs-http-client/src/ls.js +++ b/packages/ipfs-http-client/src/ls.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('./lib/configure') const toUrlSearchParams = require('./lib/to-url-search-params') @@ -11,7 +10,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(path) ? new CID(path) : path}`, + arg: `${path instanceof Uint8Array ? new CID(path) : path}`, ...options }), headers: options.headers diff --git a/packages/ipfs-http-client/src/object/data.js b/packages/ipfs-http-client/src/object/data.js index 39446a9934..1987a3d391 100644 --- a/packages/ipfs-http-client/src/object/data.js +++ b/packages/ipfs-http-client/src/object/data.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') @@ -11,13 +10,13 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`, ...options }), headers: options.headers }) const data = await res.arrayBuffer() - return Buffer.from(data) + return new Uint8Array(data, data.byteOffset, data.byteLength) } }) diff --git a/packages/ipfs-http-client/src/object/get.js b/packages/ipfs-http-client/src/object/get.js index 0ad60f7bff..4857fe724a 100644 --- a/packages/ipfs-http-client/src/object/get.js +++ b/packages/ipfs-http-client/src/object/get.js @@ -1,10 +1,10 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const { DAGNode, DAGLink } = require('ipld-dag-pb') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') +const uint8ArrayFromString = require('uint8arrays/from-string') module.exports = configure(api => { return async (cid, options = {}) => { @@ -12,7 +12,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`, dataEncoding: 'base64', ...options }), @@ -21,7 +21,7 @@ module.exports = configure(api => { const data = await res.json() return new DAGNode( - Buffer.from(data.Data, 'base64'), + uint8ArrayFromString(data.Data, 'base64pad'), (data.Links || []).map(l => new DAGLink(l.Name, l.Size, l.Hash)) ) } diff --git a/packages/ipfs-http-client/src/object/links.js b/packages/ipfs-http-client/src/object/links.js index abe71de547..97f2f5b331 100644 --- a/packages/ipfs-http-client/src/object/links.js +++ b/packages/ipfs-http-client/src/object/links.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const { DAGLink } = require('ipld-dag-pb') const configure = require('../lib/configure') @@ -12,7 +11,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`, ...options }), headers: options.headers diff --git a/packages/ipfs-http-client/src/object/patch/add-link.js b/packages/ipfs-http-client/src/object/patch/add-link.js index f9f6eb8bf2..ab8535406d 100644 --- a/packages/ipfs-http-client/src/object/patch/add-link.js +++ b/packages/ipfs-http-client/src/object/patch/add-link.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('../../lib/configure') const toUrlSearchParams = require('../../lib/to-url-search-params') @@ -12,7 +11,7 @@ module.exports = configure(api => { signal: options.signal, searchParams: toUrlSearchParams({ arg: [ - `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + `${cid instanceof Uint8Array ? new CID(cid) : cid}`, dLink.Name || dLink.name || '', (dLink.Hash || dLink.cid || '').toString() || null ], diff --git a/packages/ipfs-http-client/src/object/patch/append-data.js b/packages/ipfs-http-client/src/object/patch/append-data.js index 0ef14f9a54..c334907508 100644 --- a/packages/ipfs-http-client/src/object/patch/append-data.js +++ b/packages/ipfs-http-client/src/object/patch/append-data.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const multipartRequest = require('../../lib/multipart-request') const configure = require('../../lib/configure') @@ -18,7 +17,7 @@ module.exports = configure(api => { timeout: options.timeout, signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`, ...options }), ...( diff --git a/packages/ipfs-http-client/src/object/patch/rm-link.js b/packages/ipfs-http-client/src/object/patch/rm-link.js index 89f02b77b7..3a58238c08 100644 --- a/packages/ipfs-http-client/src/object/patch/rm-link.js +++ b/packages/ipfs-http-client/src/object/patch/rm-link.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('../../lib/configure') const toUrlSearchParams = require('../../lib/to-url-search-params') @@ -12,7 +11,7 @@ module.exports = configure(api => { signal: options.signal, searchParams: toUrlSearchParams({ arg: [ - `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + `${cid instanceof Uint8Array ? new CID(cid) : cid}`, dLink.Name || dLink.name || null ], ...options diff --git a/packages/ipfs-http-client/src/object/patch/set-data.js b/packages/ipfs-http-client/src/object/patch/set-data.js index 9609dff3a2..59c3394b52 100644 --- a/packages/ipfs-http-client/src/object/patch/set-data.js +++ b/packages/ipfs-http-client/src/object/patch/set-data.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const multipartRequest = require('../../lib/multipart-request') const configure = require('../../lib/configure') @@ -19,7 +18,7 @@ module.exports = configure(api => { signal, searchParams: toUrlSearchParams({ arg: [ - `${Buffer.isBuffer(cid) ? new CID(cid) : cid}` + `${cid instanceof Uint8Array ? new CID(cid) : cid}` ], ...options }), diff --git a/packages/ipfs-http-client/src/object/put.js b/packages/ipfs-http-client/src/object/put.js index 34bf8eebe5..c9e46898a4 100644 --- a/packages/ipfs-http-client/src/object/put.js +++ b/packages/ipfs-http-client/src/object/put.js @@ -2,12 +2,13 @@ const CID = require('cids') const { DAGNode } = require('ipld-dag-pb') -const { Buffer } = require('buffer') const multipartRequest = require('../lib/multipart-request') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') const anySignal = require('any-signal') const AbortController = require('abort-controller') +const unit8ArrayToString = require('uint8arrays/to-string') +const uint8ArrayFromString = require('uint8arrays/from-string') module.exports = configure(api => { return async (obj, options = {}) => { @@ -16,16 +17,16 @@ module.exports = configure(api => { Links: [] } - if (Buffer.isBuffer(obj)) { + if (obj instanceof Uint8Array) { if (!options.enc) { tmpObj = { - Data: obj.toString(), + Data: unit8ArrayToString(obj), Links: [] } } } else if (DAGNode.isDAGNode(obj)) { tmpObj = { - Data: obj.Data.toString(), + Data: unit8ArrayToString(obj.Data), Links: obj.Links.map(l => ({ Name: l.Name, Hash: l.Hash.toString(), @@ -33,18 +34,18 @@ module.exports = configure(api => { })) } } else if (typeof obj === 'object') { - tmpObj.Data = obj.Data.toString() + tmpObj.Data = unit8ArrayToString(obj.Data) tmpObj.Links = obj.Links } else { throw new Error('obj not recognized') } let buf - if (Buffer.isBuffer(obj) && options.enc) { + if (obj instanceof Uint8Array && options.enc) { buf = obj } else { options.enc = 'json' - buf = Buffer.from(JSON.stringify(tmpObj)) + buf = uint8ArrayFromString(JSON.stringify(tmpObj)) } // allow aborting requests on body errors diff --git a/packages/ipfs-http-client/src/object/stat.js b/packages/ipfs-http-client/src/object/stat.js index 1b8bfeb878..294a1c222b 100644 --- a/packages/ipfs-http-client/src/object/stat.js +++ b/packages/ipfs-http-client/src/object/stat.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') @@ -11,7 +10,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`, ...options }), headers: options.headers diff --git a/packages/ipfs-http-client/src/pubsub/subscribe.js b/packages/ipfs-http-client/src/pubsub/subscribe.js index fdf4d24ad3..9cd1215507 100644 --- a/packages/ipfs-http-client/src/pubsub/subscribe.js +++ b/packages/ipfs-http-client/src/pubsub/subscribe.js @@ -1,7 +1,7 @@ 'use strict' -const multibase = require('multibase') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require('uint8arrays/to-string') const log = require('debug')('ipfs-http-client:pubsub:subscribe') const SubscriptionTracker = require('./subscription-tracker') const configure = require('../lib/configure') @@ -22,7 +22,7 @@ module.exports = configure((api, options) => { const ffWorkaround = setTimeout(async () => { log(`Publishing empty message to "${topic}" to resolve subscription request`) try { - await publish(topic, Buffer.alloc(0), options) + await publish(topic, new Uint8Array(0), options) } catch (err) { log('Failed to publish empty message', err) } @@ -60,9 +60,9 @@ async function readMessages (msgStream, { onMessage, onEnd, onError }) { for await (const msg of msgStream) { try { onMessage({ - from: multibase.encode('base58btc', Buffer.from(msg.from, 'base64')).toString().slice(1), - data: Buffer.from(msg.data, 'base64'), - seqno: Buffer.from(msg.seqno, 'base64'), + from: uint8ArrayToString(uint8ArrayFromString(msg.from, 'base64pad'), 'base58btc'), + data: uint8ArrayFromString(msg.data, 'base64pad'), + seqno: uint8ArrayFromString(msg.seqno, 'base64pad'), topicIDs: msg.topicIDs }) } catch (err) { diff --git a/packages/ipfs-http-client/src/refs/index.js b/packages/ipfs-http-client/src/refs/index.js index c624434c70..b21560ee5d 100644 --- a/packages/ipfs-http-client/src/refs/index.js +++ b/packages/ipfs-http-client/src/refs/index.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const toCamel = require('../lib/object-to-camel') const configure = require('../lib/configure') @@ -16,7 +15,7 @@ module.exports = configure((api, options) => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: args.map(arg => `${Buffer.isBuffer(arg) ? new CID(arg) : arg}`), + arg: args.map(arg => `${arg instanceof Uint8Array ? new CID(arg) : arg}`), ...options }), headers: options.headers, diff --git a/packages/ipfs-http-client/test/commands.spec.js b/packages/ipfs-http-client/test/commands.spec.js index 96f932b92e..9c7de185c1 100644 --- a/packages/ipfs-http-client/test/commands.spec.js +++ b/packages/ipfs-http-client/test/commands.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() describe('.commands', function () { diff --git a/packages/ipfs-http-client/test/constructor.spec.js b/packages/ipfs-http-client/test/constructor.spec.js index e0d188d0a3..7cca6332ba 100644 --- a/packages/ipfs-http-client/test/constructor.spec.js +++ b/packages/ipfs-http-client/test/constructor.spec.js @@ -2,7 +2,7 @@ 'use strict' const multiaddr = require('multiaddr') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() const ipfsClient = require('../src/index.js') const globalThis = require('ipfs-utils/src/globalthis') diff --git a/packages/ipfs-http-client/test/dag.spec.js b/packages/ipfs-http-client/test/dag.spec.js index 5b33a66371..a5f861aeb9 100644 --- a/packages/ipfs-http-client/test/dag.spec.js +++ b/packages/ipfs-http-client/test/dag.spec.js @@ -3,8 +3,8 @@ 'use strict' -const { Buffer } = require('buffer') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const uint8ArrayFromString = require('uint8arrays/from-string') +const { expect } = require('aegir/utils/chai') const ipldDagPb = require('ipld-dag-pb') const { DAGNode } = ipldDagPb const CID = require('cids') @@ -22,7 +22,7 @@ describe('.dag', function () { after(() => f.clean()) it('should be able to put and get a DAG node with format dag-pb', async () => { - const data = Buffer.from('some data') + const data = uint8ArrayFromString('some data') const node = new DAGNode(data) let cid = await ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' }) @@ -50,7 +50,7 @@ describe('.dag', function () { }) it('should be able to put and get a DAG node with format raw', async () => { - const node = Buffer.from('some data') + const node = uint8ArrayFromString('some data') let cid = await ipfs.dag.put(node, { format: 'raw', hashAlg: 'sha2-256' }) expect(cid.codec).to.equal('raw') @@ -63,7 +63,7 @@ describe('.dag', function () { }) it('should error when missing DAG resolver for multicodec from requested CID', async () => { - const block = await ipfs.block.put(Buffer.from([0, 1, 2, 3]), { + const block = await ipfs.block.put(Uint8Array.from([0, 1, 2, 3]), { cid: new CID('z8mWaJ1dZ9fH5EetPuRsj8jj26pXsgpsr') }) @@ -71,7 +71,7 @@ describe('.dag', function () { }) it('should error when putting node with esoteric format', () => { - const node = Buffer.from('some data') + const node = uint8ArrayFromString('some data') return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/Format unsupported/) }) @@ -92,7 +92,7 @@ describe('.dag', function () { } }) - const node = Buffer.from('some data') + const node = uint8ArrayFromString('some data') // error is from go-ipfs, this means the client serialized it ok await expect(ipfs2.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/no parser for format "git-raw"/) @@ -118,7 +118,7 @@ describe('.dag', function () { hashAlg: 'sha2-256' }) - const dagPbNode = new DAGNode(Buffer.alloc(0), [], 0) + const dagPbNode = new DAGNode(new Uint8Array(0), [], 0) const cid2 = await ipfs2.dag.put(dagPbNode, { format: 'dag-pb', hashAlg: 'sha2-256' diff --git a/packages/ipfs-http-client/test/diag.spec.js b/packages/ipfs-http-client/test/diag.spec.js index 1c78916cf2..60676d9a11 100644 --- a/packages/ipfs-http-client/test/diag.spec.js +++ b/packages/ipfs-http-client/test/diag.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() describe('.diag', function () { diff --git a/packages/ipfs-http-client/test/endpoint-config.spec.js b/packages/ipfs-http-client/test/endpoint-config.spec.js index f1719d895c..1c019fb0f9 100644 --- a/packages/ipfs-http-client/test/endpoint-config.spec.js +++ b/packages/ipfs-http-client/test/endpoint-config.spec.js @@ -2,7 +2,7 @@ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const ipfsClient = require('../src') describe('.getEndpointConfig', () => { diff --git a/packages/ipfs-http-client/test/exports.spec.js b/packages/ipfs-http-client/test/exports.spec.js index c3ddff17d5..1cc99c946f 100644 --- a/packages/ipfs-http-client/test/exports.spec.js +++ b/packages/ipfs-http-client/test/exports.spec.js @@ -2,18 +2,16 @@ 'use strict' const CID = require('cids') -const { Buffer } = require('buffer') const multiaddr = require('multiaddr') const multibase = require('multibase') const multicodec = require('multicodec') const multihash = require('multihashes') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const IpfsHttpClient = require('../') describe('exports', () => { it('should export the expected types and utilities', () => { - expect(IpfsHttpClient.Buffer).to.equal(Buffer) expect(IpfsHttpClient.CID).to.equal(CID) expect(IpfsHttpClient.multiaddr).to.equal(multiaddr) expect(IpfsHttpClient.multibase).to.equal(multibase) diff --git a/packages/ipfs-http-client/test/files.spec.js b/packages/ipfs-http-client/test/files.spec.js index 9ad1be1ff7..10e2d1c844 100644 --- a/packages/ipfs-http-client/test/files.spec.js +++ b/packages/ipfs-http-client/test/files.spec.js @@ -2,8 +2,8 @@ 'use strict' -const { Buffer } = require('buffer') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const uint8ArrayFromString = require('uint8arrays/from-string') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() describe('.add', function () { @@ -18,7 +18,7 @@ describe('.add', function () { after(() => f.clean()) it('should ignore metadata until https://github.com/ipfs/go-ipfs/issues/6920 is implemented', async () => { - const data = Buffer.from('some data') + const data = uint8ArrayFromString('some data') const result = await ipfs.add(data, { mode: 0o600, mtime: { diff --git a/packages/ipfs-http-client/test/get.spec.js b/packages/ipfs-http-client/test/get.spec.js index dd2513c450..4f3044e840 100644 --- a/packages/ipfs-http-client/test/get.spec.js +++ b/packages/ipfs-http-client/test/get.spec.js @@ -3,9 +3,8 @@ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const loadFixture = require('aegir/fixtures') -const { Buffer } = require('buffer') const all = require('it-all') const concat = require('it-concat') @@ -67,7 +66,7 @@ describe('.get (specific go-ipfs features)', function () { const path = `${subdir}/${filename}` const files = await all(ipfs.addAll([{ path, - content: Buffer.from(path) + content: path }])) expect(files[2].cid.toString()).to.equal(expectedCid) diff --git a/packages/ipfs-http-client/test/key.spec.js b/packages/ipfs-http-client/test/key.spec.js index eb53b8293d..95028c7446 100644 --- a/packages/ipfs-http-client/test/key.spec.js +++ b/packages/ipfs-http-client/test/key.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() describe('.key', function () { diff --git a/packages/ipfs-http-client/test/lib.error-handler.spec.js b/packages/ipfs-http-client/test/lib.error-handler.spec.js index 9804ccdd47..7e73498616 100644 --- a/packages/ipfs-http-client/test/lib.error-handler.spec.js +++ b/packages/ipfs-http-client/test/lib.error-handler.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const throwsAsync = require('./utils/throws-async') const { errorHandler, HTTPError } = require('../src/lib/core') diff --git a/packages/ipfs-http-client/test/log.spec.js b/packages/ipfs-http-client/test/log.spec.js index e41ea9b03c..814ff44887 100644 --- a/packages/ipfs-http-client/test/log.spec.js +++ b/packages/ipfs-http-client/test/log.spec.js @@ -2,8 +2,8 @@ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const { Buffer } = require('buffer') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') const f = require('./utils/factory')() describe('.log', function () { @@ -20,7 +20,7 @@ describe('.log', function () { it('.log.tail', async () => { const i = setInterval(async () => { try { - await ipfs.add(Buffer.from('just adding some data to generate logs')) + await ipfs.add(uint8ArrayFromString('just adding some data to generate logs')) } catch (_) { // this can error if the test has finished and we're shutting down the node } diff --git a/packages/ipfs-http-client/test/node/custom-headers.js b/packages/ipfs-http-client/test/node/custom-headers.js index 98d3f1a683..2d0df7a847 100644 --- a/packages/ipfs-http-client/test/node/custom-headers.js +++ b/packages/ipfs-http-client/test/node/custom-headers.js @@ -2,8 +2,9 @@ 'use strict' const { isNode } = require('ipfs-utils/src/env') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const ipfsClient = require('../../src') +const uint8ArrayFromString = require('uint8arrays/from-string') function startServer (fn) { let headersResolve @@ -59,7 +60,7 @@ describe('custom headers', function () { }) it('multipart API calls', async () => { - const headers = await startServer(() => ipfs.files.write('/foo/bar', Buffer.from('derp'), { + const headers = await startServer(() => ipfs.files.write('/foo/bar', uint8ArrayFromString('derp'), { create: true })) @@ -88,7 +89,7 @@ describe('custom headers', function () { }) it('multipart API calls', async () => { - const headers = await startServer(() => ipfs.files.write('/foo/bar', Buffer.from('derp'), { + const headers = await startServer(() => ipfs.files.write('/foo/bar', uint8ArrayFromString('derp'), { create: true, headers: { authorization: 'Bearer OLOY' diff --git a/packages/ipfs-http-client/test/node/request-api.js b/packages/ipfs-http-client/test/node/request-api.js index 422b0f0d73..ed91c61574 100644 --- a/packages/ipfs-http-client/test/node/request-api.js +++ b/packages/ipfs-http-client/test/node/request-api.js @@ -1,8 +1,8 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const { Buffer } = require('buffer') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') const ipfsClient = require('../../src/index.js') describe('\'deal with HTTP weirdness\' tests', () => { @@ -40,7 +40,7 @@ describe('trailer headers', () => { server.listen(6001, () => { const ipfs = ipfsClient('/ip4/127.0.0.1/tcp/6001') /* eslint-disable */ - ipfs.add(Buffer.from('Hello there!'), (err, res) => { + ipfs.add(uint8ArrayFromString('Hello there!'), (err, res) => { // TODO: error's are not being correctly // propagated with Trailer headers yet // expect(err).to.exist() diff --git a/packages/ipfs-http-client/test/node/swarm.js b/packages/ipfs-http-client/test/node/swarm.js index 5f4d016433..c437caabfe 100644 --- a/packages/ipfs-http-client/test/node/swarm.js +++ b/packages/ipfs-http-client/test/node/swarm.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const nock = require('nock') const ipfsClient = require('../../src') diff --git a/packages/ipfs-http-client/test/ping.spec.js b/packages/ipfs-http-client/test/ping.spec.js index b464062bbc..46f4e45888 100644 --- a/packages/ipfs-http-client/test/ping.spec.js +++ b/packages/ipfs-http-client/test/ping.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const all = require('it-all') const f = require('./utils/factory')() diff --git a/packages/ipfs-http-client/test/repo.spec.js b/packages/ipfs-http-client/test/repo.spec.js index 7a1b643d7f..bac770858b 100644 --- a/packages/ipfs-http-client/test/repo.spec.js +++ b/packages/ipfs-http-client/test/repo.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() describe('.repo', function () { diff --git a/packages/ipfs-http-client/test/stats.spec.js b/packages/ipfs-http-client/test/stats.spec.js index 6b1530beef..abae594189 100644 --- a/packages/ipfs-http-client/test/stats.spec.js +++ b/packages/ipfs-http-client/test/stats.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const all = require('it-all') const f = require('./utils/factory')() diff --git a/packages/ipfs-http-client/test/sub-modules.spec.js b/packages/ipfs-http-client/test/sub-modules.spec.js index a5db456df0..9b939d1e56 100644 --- a/packages/ipfs-http-client/test/sub-modules.spec.js +++ b/packages/ipfs-http-client/test/sub-modules.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') describe('submodules', () => { it('bitswap', () => { diff --git a/packages/ipfs-message-port-client/package.json b/packages/ipfs-message-port-client/package.json index e910f2b3ee..e1ca16a442 100644 --- a/packages/ipfs-message-port-client/package.json +++ b/packages/ipfs-message-port-client/package.json @@ -38,13 +38,13 @@ "cids": "^1.0.0" }, "devDependencies": { - "aegir": "^25.1.0", + "aegir": "^26.0.0", "cross-env": "^7.0.0", "interface-ipfs-core": "^0.139.0", "ipfs": "^0.49.0", "ipfs-message-port-protocol": "^0.1.0", "ipfs-message-port-server": "^0.1.0", - "ipld-dag-pb": "^0.19.0" + "ipld-dag-pb": "^0.20.0" }, "engines": { "node": ">=10.3.0", diff --git a/packages/ipfs-message-port-protocol/README.md b/packages/ipfs-message-port-protocol/README.md index d4d32fe9be..7ef771235a 100644 --- a/packages/ipfs-message-port-protocol/README.md +++ b/packages/ipfs-message-port-protocol/README.md @@ -18,11 +18,12 @@ - [Install](#install) - [Usage](#usage) - - [Wire protocol codecs](#wire-protocol-codecs) +- [Wire protocol codecs](#wire-protocol-codecs) + - [`CID`](#cid) - [Block](#block) - [DAGNode](#dagnode) - [AsyncIterable](#asynciterable) - - [Callback][#callback] + - [Callback](#callback) - [Contribute](#contribute) - [License](#license) @@ -72,7 +73,7 @@ Codecs for [IPLD Block][] implementation in JavaScript. ```js const { Block, encodeBlock, decodeBlock } = require('ipfs-message-port-protocol/src/block') -const data = Buffer.from('hello') +const data = new TextEncoder().encode('hello') const cid = new CID('QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n') const block = new Block(data, cid) diff --git a/packages/ipfs-message-port-protocol/package.json b/packages/ipfs-message-port-protocol/package.json index 1e7e9f4cb3..6d41c91d0a 100644 --- a/packages/ipfs-message-port-protocol/package.json +++ b/packages/ipfs-message-port-protocol/package.json @@ -32,12 +32,11 @@ "dep-check": "aegir dep-check" }, "dependencies": { - "buffer": "^5.6.0", "cids": "^1.0.0", "ipld-block": "^0.10.0" }, "devDependencies": { - "aegir": "^25.0.0", + "aegir": "^26.0.0", "interface-ipfs-core": "^0.139.0" }, "engines": { diff --git a/packages/ipfs-message-port-protocol/src/cid.js b/packages/ipfs-message-port-protocol/src/cid.js index a17f60bc67..a79895301b 100644 --- a/packages/ipfs-message-port-protocol/src/cid.js +++ b/packages/ipfs-message-port-protocol/src/cid.js @@ -1,7 +1,6 @@ 'use strict' const CID = require('cids') -const { Buffer } = require('buffer') /** * @typedef {Object} EncodedCID diff --git a/packages/ipfs-message-port-protocol/src/rpc.ts b/packages/ipfs-message-port-protocol/src/rpc.ts index 1b9dd836de..51c9c045e8 100644 --- a/packages/ipfs-message-port-protocol/src/rpc.ts +++ b/packages/ipfs-message-port-protocol/src/rpc.ts @@ -2,7 +2,7 @@ export type Procedure = T extends (arg: infer I) => infer O ? (query: I & QueryOptions) => Return : void -export type Remote = { +export type Remote> = { [K in keyof T]: Procedure } @@ -22,41 +22,37 @@ export type TransferOptions = { export type NonUndefined = A extends undefined ? never : A -export type ProcedureNames = { - [K in keyof T]-?: NonUndefined extends Function ? K : never +export type ProcedureNames> = { + [K in keyof T]-?: NonUndefined extends Function ? K : never // eslint-disable-line @typescript-eslint/ban-types }[keyof T][] - - /** * Any method name of the associated with RPC service. */ -export type Method = ServiceQuery['method'] +export type Method> = ServiceQuery['method'] /** * Namespace of the RCP service */ -export type Namespace = ServiceQuery['namespace'] +export type Namespace> = ServiceQuery['namespace'] -export type Values = T[keyof T] -export type Keys = keyof T +export type Values> = T[keyof T] +export type Keys> = keyof T -export type Inn = ServiceQuery['input'] -export type Out = ServiceQuery['result'] +export type Inn> = ServiceQuery['input'] +export type Out> = ServiceQuery['result'] -export type RPCQuery = Pick< +export type RPCQuery> = Pick< ServiceQuery, 'method' | 'namespace' | 'input' | 'timeout' | 'signal' > - export type ServiceQuery = Values< { [NS in keyof T]: NamespacedQuery } > - export type NamespacedQuery = Values< { [M in keyof S]-?: S[M] extends (input: infer I) => infer O @@ -74,8 +70,7 @@ type R = O extends Promise ? Promise> : Promise> -type WithTransferOptions = O extends object ? O & TransferOptions : O - +type WithTransferOptions = O extends Record ? O & TransferOptions : O export type MultiService = { [NS in keyof T]: NamespacedService diff --git a/packages/ipfs-message-port-protocol/test/block.browser.js b/packages/ipfs-message-port-protocol/test/block.browser.js index e769f5c0c3..ed7acc58a2 100644 --- a/packages/ipfs-message-port-protocol/test/block.browser.js +++ b/packages/ipfs-message-port-protocol/test/block.browser.js @@ -5,8 +5,8 @@ const CID = require('cids') const { encodeBlock, decodeBlock } = require('../src/block') const { ipc } = require('./util') -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const { Buffer } = require('buffer') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') const Block = require('ipld-block') describe('block (browser)', function () { @@ -16,7 +16,7 @@ describe('block (browser)', function () { describe('encodeBlock / decodeBlock', () => { it('should decode Block over message channel', async () => { const blockIn = new Block( - Buffer.from('hello'), + uint8ArrayFromString('hello'), new CID('QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n') ) @@ -27,7 +27,7 @@ describe('block (browser)', function () { it('should decode Block over message channel & transfer bytes', async () => { const cid = new CID('QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n') - const data = Buffer.from('hello') + const data = uint8ArrayFromString('hello') const blockIn = new Block(data, cid) const transfer = [] @@ -39,7 +39,7 @@ describe('block (browser)', function () { expect(blockOut).to.be.instanceOf(Block) expect(blockOut).to.be.deep.equal( new Block( - Buffer.from('hello'), + uint8ArrayFromString('hello'), new CID('QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n') ) ) diff --git a/packages/ipfs-message-port-protocol/test/cid.browser.js b/packages/ipfs-message-port-protocol/test/cid.browser.js index ece6fd75ec..3b4761127f 100644 --- a/packages/ipfs-message-port-protocol/test/cid.browser.js +++ b/packages/ipfs-message-port-protocol/test/cid.browser.js @@ -5,7 +5,7 @@ const CID = require('cids') const { encodeCID, decodeCID } = require('../src/cid') const { ipc } = require('./util') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') describe('cid (browser)', function () { this.timeout(10 * 1000) diff --git a/packages/ipfs-message-port-protocol/test/cid.spec.js b/packages/ipfs-message-port-protocol/test/cid.spec.js index d6aec85304..14bee372f8 100644 --- a/packages/ipfs-message-port-protocol/test/cid.spec.js +++ b/packages/ipfs-message-port-protocol/test/cid.spec.js @@ -4,7 +4,7 @@ const CID = require('cids') const { encodeCID, decodeCID } = require('../src/cid') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') describe('cid', function () { this.timeout(10 * 1000) diff --git a/packages/ipfs-message-port-protocol/test/core.browser.js b/packages/ipfs-message-port-protocol/test/core.browser.js index 3f49a977d6..ae2ea46b56 100644 --- a/packages/ipfs-message-port-protocol/test/core.browser.js +++ b/packages/ipfs-message-port-protocol/test/core.browser.js @@ -9,8 +9,8 @@ const { decodeIterable } = require('../src/core') const { ipc } = require('./util') -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const { Buffer } = require('buffer') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') describe('core', function () { this.timeout(10 * 1000) @@ -55,13 +55,13 @@ describe('core', function () { await move(encodeCallback(callback, transfer), transfer) ) - remote({ hello: Buffer.from('world') }) - expect(await receive()).to.be.deep.equal({ hello: Buffer.from('world') }) + remote({ hello: uint8ArrayFromString('world') }) + expect(await receive()).to.be.deep.equal({ hello: uint8ArrayFromString('world') }) - const world = Buffer.from('world') + const world = uint8ArrayFromString('world') remote({ hello: world }, [world.buffer]) - expect(await receive()).to.be.deep.equal({ hello: Buffer.from('world') }) + expect(await receive()).to.be.deep.equal({ hello: uint8ArrayFromString('world') }) expect(world.buffer).property('byteLength', 0, 'buffer was cleared') }) }) @@ -71,8 +71,8 @@ describe('core', function () { const iterate = async function * () { yield 1 await null - yield { hello: Buffer.from('world') } - yield { items: [Buffer.from('bla'), Buffer.from('bla')] } + yield { hello: uint8ArrayFromString('world') } + yield { items: [uint8ArrayFromString('bla'), uint8ArrayFromString('bla')] } } const transfer = [] @@ -93,8 +93,8 @@ describe('core', function () { const incoming = [ 1, - { hello: Buffer.from('world') }, - { items: [Buffer.from('bla'), Buffer.from('bla')] } + { hello: uint8ArrayFromString('world') }, + { items: [uint8ArrayFromString('bla'), uint8ArrayFromString('bla')] } ] for await (const item of remote) { @@ -107,8 +107,8 @@ describe('core', function () { it('break in consumer loop propagates to producer loop', async () => { const outgoing = [ 1, - { hello: Buffer.from('world') }, - { items: [Buffer.from('bla'), Buffer.from('bla')] }, + { hello: uint8ArrayFromString('world') }, + { items: [uint8ArrayFromString('bla'), uint8ArrayFromString('bla')] }, { bye: 'Goodbye' } ] @@ -137,8 +137,8 @@ describe('core', function () { const incoming = [ 1, - { hello: Buffer.from('world') }, - { items: [Buffer.from('bla'), Buffer.from('bla')] } + { hello: uint8ArrayFromString('world') }, + { items: [uint8ArrayFromString('bla'), uint8ArrayFromString('bla')] } ] for await (const item of remote) { @@ -237,9 +237,9 @@ describe('core', function () { }) it('iterable transfers yield data', async () => { - const hi = Buffer.from('hello world') - const body = Buffer.from('how are you') - const bye = Buffer.from('Bye') + const hi = uint8ArrayFromString('hello world') + const body = uint8ArrayFromString('how are you') + const bye = uint8ArrayFromString('Bye') const outgoing = [hi, body, bye] const iterate = async function * () { await null @@ -264,9 +264,9 @@ describe('core', function () { ) const incoming = [ - Buffer.from('hello world'), - Buffer.from('how are you'), - Buffer.from('Bye') + uint8ArrayFromString('hello world'), + uint8ArrayFromString('how are you'), + uint8ArrayFromString('Bye') ] for await (const data of remote) { @@ -284,8 +284,8 @@ describe('core', function () { it('remote iterable copies yielded data', async () => { const iterate = function * () { yield 1 - yield { hello: Buffer.from('world') } - yield { items: [Buffer.from('bla'), Buffer.from('bla')] } + yield { hello: uint8ArrayFromString('world') } + yield { items: [uint8ArrayFromString('bla'), uint8ArrayFromString('bla')] } } const transfer = [] @@ -306,8 +306,8 @@ describe('core', function () { const incoming = [ 1, - { hello: Buffer.from('world') }, - { items: [Buffer.from('bla'), Buffer.from('bla')] } + { hello: uint8ArrayFromString('world') }, + { items: [uint8ArrayFromString('bla'), uint8ArrayFromString('bla')] } ] for await (const item of remote) { @@ -320,8 +320,8 @@ describe('core', function () { it('break in consumer loop propagates to producer loop', async () => { const outgoing = [ 1, - { hello: Buffer.from('world') }, - { items: [Buffer.from('bla'), Buffer.from('bla')] }, + { hello: uint8ArrayFromString('world') }, + { items: [uint8ArrayFromString('bla'), uint8ArrayFromString('bla')] }, { bye: 'Goodbye' } ] @@ -350,8 +350,8 @@ describe('core', function () { const incoming = [ 1, - { hello: Buffer.from('world') }, - { items: [Buffer.from('bla'), Buffer.from('bla')] } + { hello: uint8ArrayFromString('world') }, + { items: [uint8ArrayFromString('bla'), uint8ArrayFromString('bla')] } ] for await (const item of remote) { @@ -448,9 +448,9 @@ describe('core', function () { }) it('iterable transfers yield data', async () => { - const hi = Buffer.from('hello world') - const body = Buffer.from('how are you') - const bye = Buffer.from('Bye') + const hi = uint8ArrayFromString('hello world') + const body = uint8ArrayFromString('how are you') + const bye = uint8ArrayFromString('Bye') const outgoing = [hi, body, bye] const iterate = function * () { yield * outgoing @@ -474,9 +474,9 @@ describe('core', function () { ) const incoming = [ - Buffer.from('hello world'), - Buffer.from('how are you'), - Buffer.from('Bye') + uint8ArrayFromString('hello world'), + uint8ArrayFromString('how are you'), + uint8ArrayFromString('Bye') ] for await (const data of remote) { diff --git a/packages/ipfs-message-port-protocol/test/dag.browser.js b/packages/ipfs-message-port-protocol/test/dag.browser.js index a522b3c1d6..34451b858f 100644 --- a/packages/ipfs-message-port-protocol/test/dag.browser.js +++ b/packages/ipfs-message-port-protocol/test/dag.browser.js @@ -5,8 +5,8 @@ const CID = require('cids') const { encodeNode, decodeNode } = require('../src/dag') const { ipc } = require('./util') -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const { Buffer } = require('buffer') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') describe('dag (browser)', function () { this.timeout(10 * 1000) @@ -19,7 +19,7 @@ describe('dag (browser)', function () { ) const cid2 = new CID('QmPv52ekjS75L4JmHpXVeuJ5uX2ecSfSZo88NSyxwA3rAQ') - const hi = Buffer.from('hello world') + const hi = uint8ArrayFromString('hello world') const nodeIn = { hi, nested: { @@ -45,9 +45,9 @@ describe('dag (browser)', function () { ) const cid2 = new CID('QmPv52ekjS75L4JmHpXVeuJ5uX2ecSfSZo88NSyxwA3rAQ') - const hi = Buffer.from('hello world') + const hi = uint8ArrayFromString('hello world') const nodeIn = { - hi: Buffer.from(hi), + hi, nested: { structure: { with: { diff --git a/packages/ipfs-message-port-protocol/test/dag.spec.js b/packages/ipfs-message-port-protocol/test/dag.spec.js index 53c0388ac3..4a2ae57b7d 100644 --- a/packages/ipfs-message-port-protocol/test/dag.spec.js +++ b/packages/ipfs-message-port-protocol/test/dag.spec.js @@ -4,8 +4,8 @@ const CID = require('cids') const { encodeNode } = require('../src/dag') -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const { Buffer } = require('buffer') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') describe('dag', function () { this.timeout(10 * 1000) @@ -41,7 +41,7 @@ describe('dag', function () { ) const cid2 = new CID('QmPv52ekjS75L4JmHpXVeuJ5uX2ecSfSZo88NSyxwA3rAQ') - const hi = Buffer.from('hello world') + const hi = uint8ArrayFromString('hello world') const dagNode = { hi, nested: { @@ -78,7 +78,7 @@ describe('dag', function () { ) const cid2 = new CID('QmPv52ekjS75L4JmHpXVeuJ5uX2ecSfSZo88NSyxwA3rAQ') - const hi = Buffer.from('hello world') + const hi = uint8ArrayFromString('hello world') const dagNode = { hi, nested: { diff --git a/packages/ipfs-message-port-server/package.json b/packages/ipfs-message-port-server/package.json index aa3fd79e75..12e41fd1a4 100644 --- a/packages/ipfs-message-port-server/package.json +++ b/packages/ipfs-message-port-server/package.json @@ -39,7 +39,7 @@ }, "devDependencies": { "@types/it-all": "^1.0.0", - "aegir": "^25.0.0", + "aegir": "^26.0.0", "cross-env": "^7.0.0", "interface-ipfs-core": "^0.139.0", "ipfs": "^0.49.0", diff --git a/packages/ipfs-message-port-server/src/block.js b/packages/ipfs-message-port-server/src/block.js index 5078ad5a61..ca24aca886 100644 --- a/packages/ipfs-message-port-server/src/block.js +++ b/packages/ipfs-message-port-server/src/block.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const collect = require('it-all') const { encodeError } = require('ipfs-message-port-protocol/src/error') const { decodeCID, encodeCID } = require('ipfs-message-port-protocol/src/cid') @@ -70,11 +69,8 @@ class BlockService { */ async put (query) { const input = query.block - /** @type {Buffer|Block} */ - const block = - input instanceof Uint8Array - ? Buffer.from(input.buffer, input.byteOffset, input.byteLength) - : decodeBlock(input) + /** @type {Uint8Array|Block} */ + const block = input instanceof Uint8Array ? input : decodeBlock(input) const result = await this.ipfs.block.put(block, { ...query, cid: query.cid ? decodeCID(query.cid) : query.cid diff --git a/packages/ipfs-message-port-server/src/ipfs.ts b/packages/ipfs-message-port-server/src/ipfs.ts index 1963b17bcb..ce62969bd6 100644 --- a/packages/ipfs-message-port-server/src/ipfs.ts +++ b/packages/ipfs-message-port-server/src/ipfs.ts @@ -161,7 +161,6 @@ type AddInput = | FileInput | ReadStream - type AddAllInput = | Iterable | AsyncIterable @@ -219,7 +218,7 @@ interface BlockService { ): Promise<{ cid: CID; size: number }> } -interface GetBlockOptions extends AbortOptions { } +interface GetBlockOptions extends AbortOptions { } // eslint-disable-line @typescript-eslint/no-empty-interface interface PutBlockOptions extends AbortOptions { format?: string mhtype?: string @@ -236,4 +235,4 @@ interface RmBlockOptions extends AbortOptions { quiet?: boolean } -interface StatBlockOptions extends AbortOptions { } +interface StatBlockOptions extends AbortOptions { } // eslint-disable-line @typescript-eslint/no-empty-interface diff --git a/packages/ipfs-message-port-server/test/basic.spec.js b/packages/ipfs-message-port-server/test/basic.spec.js index fc90b4899d..4c94264945 100644 --- a/packages/ipfs-message-port-server/test/basic.spec.js +++ b/packages/ipfs-message-port-server/test/basic.spec.js @@ -3,7 +3,7 @@ /* eslint-env mocha */ const { Server } = require('../src/server') const { IPFSService } = require('../src/index') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') describe('dag', function () { this.timeout(10 * 1000) diff --git a/packages/ipfs/.aegir.js b/packages/ipfs/.aegir.js index a7d6d937c8..f482c1e420 100644 --- a/packages/ipfs/.aegir.js +++ b/packages/ipfs/.aegir.js @@ -25,6 +25,15 @@ module.exports = { }], browserNoActivityTimeout: 100 * 1000 }, + webpack: { + node: { + // required by the nofilter module + stream: true, + + // required by the core-util-is module + Buffer: true + } + }, hooks: { node: { pre: async () => { diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index f0af0303f4..6931b015cf 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -73,13 +73,12 @@ "bignumber.js": "^9.0.0", "bl": "^4.0.2", "bs58": "^4.0.1", - "buffer": "^5.6.0", "byteman": "^1.3.5", "cbor": "^5.0.1", - "cid-tool": "^0.4.0", + "cid-tool": "multiformats/js-cid-tool#fix/replace-node-buffers-with-uint8arrays", "cids": "^1.0.0", "class-is": "^1.1.0", - "dag-cbor-links": "^1.3.3", + "dag-cbor-links": "achingbrain/dag-cbor-links#chore/update-deps", "datastore-core": "^2.0.0", "datastore-pubsub": "^0.4.0", "debug": "^4.1.0", @@ -93,16 +92,16 @@ "hapi-pino": "^6.1.0", "hashlru": "^2.3.0", "interface-datastore": "^2.0.0", - "ipfs-bitswap": "^2.0.1", + "ipfs-bitswap": "ipfs/js-ipfs-bitswap#fix/replace-node-buffers-with-uint8arrays", "ipfs-block-service": "^0.18.0", - "ipfs-core-utils": "^0.3.0", - "ipfs-http-client": "^45.0.0", - "ipfs-http-response": "^0.5.0", + "ipfs-core-utils": "^0.3.1", + "ipfs-http-client": "^46.0.0", + "ipfs-http-response": "^0.6.0", "ipfs-repo": "^6.0.1", "ipfs-unixfs": "^2.0.2", "ipfs-unixfs-exporter": "^3.0.2", "ipfs-unixfs-importer": "^3.0.2", - "ipfs-utils": "^2.2.2", + "ipfs-utils": "ipfs/js-ipfs-utils#revert-54-xhr", "ipld": "^0.27.1", "ipld-bitcoin": "^0.4.0", "ipld-block": "^0.10.0", @@ -112,7 +111,7 @@ "ipld-git": "^0.6.1", "ipld-raw": "^6.0.0", "ipld-zcash": "^0.5.0", - "ipns": "^0.7.1", + "ipns": "^0.8.0", "is-domain-name": "^1.0.1", "is-ipfs": "^2.0.0", "iso-url": "^0.4.7", @@ -130,27 +129,27 @@ "iterable-ndjson": "^1.1.0", "jsondiffpatch": "^0.4.1", "just-safe-set": "^2.1.0", - "libp2p": "^0.28.10", + "libp2p": "libp2p/js-libp2p#fix/replace-node-buffers-with-uint8arrays", "libp2p-bootstrap": "^0.12.0", "libp2p-crypto": "^0.18.0", - "libp2p-delegated-content-routing": "^0.5.0", - "libp2p-delegated-peer-routing": "^0.5.0", + "libp2p-delegated-content-routing": "^0.6.0", + "libp2p-delegated-peer-routing": "^0.6.0", "libp2p-floodsub": "^0.22.0", - "libp2p-gossipsub": "^0.4.0", + "libp2p-gossipsub": "^0.5.0", "libp2p-kad-dht": "^0.20.0", - "libp2p-mdns": "^0.14.1", - "libp2p-mplex": "^0.9.5", - "libp2p-noise": "^1.1.1", + "libp2p-mdns": "^0.15.0", + "libp2p-mplex": "^0.10.0", + "libp2p-noise": "npm:@achingbrain/libp2p-noise@1.1.2", "libp2p-record": "^0.9.0", "libp2p-secio": "^0.13.0", "libp2p-tcp": "^0.15.0", - "libp2p-webrtc-star": "^0.18.0", - "libp2p-websockets": "^0.13.3", + "libp2p-webrtc-star": "^0.19.0", + "libp2p-websockets": "^0.14.0", "mafmt": "^8.0.0", "merge-options": "^2.0.0", "mortice": "^2.0.0", "multiaddr": "^8.0.0", - "multiaddr-to-uri": "^5.1.0", + "multiaddr-to-uri": "^6.0.0", "multibase": "^3.0.0", "multicodec": "^2.0.0", "multihashing-async": "^2.0.1", @@ -166,15 +165,15 @@ "streaming-iterables": "^5.0.0", "temp": "^0.9.0", "timeout-abort-controller": "^1.1.0", + "uint8arrays": "^1.1.0", "update-notifier": "^4.0.0", - "uri-to-multiaddr": "^3.0.2", + "uri-to-multiaddr": "^4.0.0", "varint": "^5.0.0", "yargs": "^15.1.0", "yargs-promise": "^1.1.0" }, "devDependencies": { - "aegir": "^25.0.0", - "base64url": "^3.0.1", + "aegir": "^26.0.0", "clear-module": "^4.0.0", "cross-env": "^7.0.0", "delay": "^4.3.0", @@ -182,7 +181,7 @@ "go-ipfs": "^0.6.0", "interface-ipfs-core": "^0.139.0", "ipfs-interop": "github:ipfs/interop#refactor/streaming-pin-api", - "ipfsd-ctl": "^5.0.0", + "ipfsd-ctl": "^6.0.0", "iso-random-stream": "^1.1.1", "it-to-buffer": "^1.0.0", "nanoid": "^3.0.2", diff --git a/packages/ipfs/src/cli/commands/block/get.js b/packages/ipfs/src/cli/commands/block/get.js index eeb36a5bdd..70f5f664fd 100644 --- a/packages/ipfs/src/cli/commands/block/get.js +++ b/packages/ipfs/src/cli/commands/block/get.js @@ -1,6 +1,7 @@ 'use strict' const parseDuration = require('parse-duration').default +const uint8ArrayToString = require('uint8arrays/to-string') module.exports = { command: 'get ', @@ -20,7 +21,7 @@ module.exports = { timeout }) if (block) { - print(block.data, false) + print(uint8ArrayToString(block.data), false) } else { print('Block was unwanted before it could be remotely retrieved') } diff --git a/packages/ipfs/src/cli/commands/cat.js b/packages/ipfs/src/cli/commands/cat.js index 2465182ffb..4c98aa65df 100644 --- a/packages/ipfs/src/cli/commands/cat.js +++ b/packages/ipfs/src/cli/commands/cat.js @@ -1,6 +1,7 @@ 'use strict' const parseDuration = require('parse-duration').default +const uint8ArrayToString = require('uint8arrays/to-string') module.exports = { command: 'cat ', @@ -26,7 +27,7 @@ module.exports = { async handler ({ ctx: { ipfs, print }, ipfsPath, offset, length, timeout }) { for await (const buf of ipfs.cat(ipfsPath, { offset, length, timeout })) { - print.write(buf) + print.write(uint8ArrayToString(buf)) } } } diff --git a/packages/ipfs/src/cli/commands/dag/get.js b/packages/ipfs/src/cli/commands/dag/get.js index 8506bd5a20..65be4925d3 100644 --- a/packages/ipfs/src/cli/commands/dag/get.js +++ b/packages/ipfs/src/cli/commands/dag/get.js @@ -1,8 +1,8 @@ 'use strict' const parseDuration = require('parse-duration').default -const { Buffer } = require('buffer') const toCidAndPath = require('ipfs-core-utils/src/to-cid-and-path') +const uint8ArrayToString = require('uint8arrays/to-string') module.exports = { command: 'get ', @@ -52,13 +52,13 @@ module.exports = { // * reads as 'agree in' if (node._json) { delete node._json.multihash - node._json.data = '0x' + node._json.data.toString('hex') + node._json.data = '0x' + uint8ArrayToString(node._json.data, 'base16') print(JSON.stringify(node._json, null, 4)) return } - if (Buffer.isBuffer(node)) { - print('0x' + node.toString('hex')) + if (node instanceof Uint8Array) { + print('0x' + uint8ArrayToString(node, 'base16')) return } diff --git a/packages/ipfs/src/cli/commands/dag/put.js b/packages/ipfs/src/cli/commands/dag/put.js index f2d4edf182..3be3d87e6f 100644 --- a/packages/ipfs/src/cli/commands/dag/put.js +++ b/packages/ipfs/src/cli/commands/dag/put.js @@ -1,5 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') + const mh = require('multihashing-async').multihash const multibase = require('multibase') const dagCBOR = require('ipld-dag-cbor') @@ -8,9 +8,10 @@ const concat = require('it-concat') const CID = require('cids') const { cidToString } = require('../../../utils/cid') const parseDuration = require('parse-duration').default +const uint8ArrayToString = require('uint8arrays/to-string') const inputDecoders = { - json: (buf) => JSON.parse(buf.toString()), + json: (buf) => JSON.parse(uint8ArrayToString(buf)), cbor: (buf) => dagCBOR.util.deserialize(buf), protobuf: (buf) => dagPB.util.deserialize(buf), raw: (buf) => buf diff --git a/packages/ipfs/src/cli/commands/object/get.js b/packages/ipfs/src/cli/commands/object/get.js index ee4b0c83c0..c9618d0055 100644 --- a/packages/ipfs/src/cli/commands/object/get.js +++ b/packages/ipfs/src/cli/commands/object/get.js @@ -3,7 +3,7 @@ const multibase = require('multibase') const { cidToString } = require('../../../utils/cid') const parseDuration = require('parse-duration').default -const { Buffer } = require('buffer') +const uint8ArrayToString = require('uint8arrays/to-string') module.exports = { command: 'get ', @@ -30,8 +30,8 @@ module.exports = { const node = await ipfs.object.get(key, { enc: 'base58', timeout }) let data = node.Data || '' - if (Buffer.isBuffer(data)) { - data = node.Data.toString(dataEncoding || undefined) + if (data instanceof Uint8Array) { + data = uint8ArrayToString(node.Data, dataEncoding || undefined) } const answer = { diff --git a/packages/ipfs/src/cli/commands/pubsub/pub.js b/packages/ipfs/src/cli/commands/pubsub/pub.js index e54a68b753..5234630551 100644 --- a/packages/ipfs/src/cli/commands/pubsub/pub.js +++ b/packages/ipfs/src/cli/commands/pubsub/pub.js @@ -1,7 +1,7 @@ 'use strict' const parseDuration = require('parse-duration').default -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') module.exports = { command: 'pub ', @@ -16,7 +16,7 @@ module.exports = { }, async handler ({ ctx: { ipfs }, topic, data, timeout }) { - data = Buffer.from(String(data)) + data = uint8ArrayFromString(String(data)) await ipfs.pubsub.publish(topic, data, { timeout }) diff --git a/packages/ipfs/src/cli/commands/refs-local.js b/packages/ipfs/src/cli/commands/refs-local.js index bdbe575b63..ac86abeb0e 100644 --- a/packages/ipfs/src/cli/commands/refs-local.js +++ b/packages/ipfs/src/cli/commands/refs-local.js @@ -1,8 +1,8 @@ 'use strict' +const uint8ArrayFromString = require('uint8arrays/from-string') const parseDuration = require('parse-duration').default const multibase = require('multibase') -const { Buffer } = require('buffer') module.exports = { command: 'refs-local', @@ -31,7 +31,7 @@ module.exports = { print(err, true, true) } else { if (multihash) { - print(multibase.encoding('base32upper').encode(Buffer.from(ref))) + print(multibase.encoding('base32upper').encode(uint8ArrayFromString(ref))) } else { print(ref) } diff --git a/packages/ipfs/src/core/components/dht.js b/packages/ipfs/src/core/components/dht.js index b249f4bb78..fb2954cfb3 100644 --- a/packages/ipfs/src/core/components/dht.js +++ b/packages/ipfs/src/core/components/dht.js @@ -4,28 +4,27 @@ const PeerId = require('peer-id') const CID = require('cids') const errCode = require('err-code') const { withTimeoutOption } = require('../utils') -const { Buffer } = require('buffer') module.exports = ({ libp2p, repo }) => { return { /** * Given a key, query the DHT for its best value. * - * @param {Buffer} key + * @param {Uint8Array} key * @param {Object} [options] - get options * @param {number} [options.timeout] - optional timeout - * @returns {Promise} + * @returns {Promise} */ get: withTimeoutOption(async (key, options) => { // eslint-disable-line require-await options = options || {} - if (!Buffer.isBuffer(key)) { + if (!(key instanceof Uint8Array)) { try { key = key.toString().split('/') .filter(part => part && part !== 'ipfs' && part !== 'ipns') .shift() - key = (new CID(key)).buffer + key = (new CID(key)).bytes } catch (err) { throw errCode(err, 'ERR_INVALID_CID') } @@ -41,18 +40,18 @@ module.exports = ({ libp2p, repo }) => { * form, this will write that value to the DHT with * that key. * - * @param {Buffer} key - * @param {Buffer} value + * @param {Uint8Array} key + * @param {Uint8Array} value * @returns {Promise} */ put: withTimeoutOption(async (key, value) => { // eslint-disable-line require-await - if (!Buffer.isBuffer(key)) { + if (!(key instanceof Uint8Array)) { try { key = key.toString().split('/') .filter(part => part && part !== 'ipfs' && part !== 'ipns') .shift() - key = (new CID(key)).buffer + key = (new CID(key)).bytes } catch (err) { throw errCode(err, 'ERR_INVALID_CID') } diff --git a/packages/ipfs/src/core/components/files/utils/hamt-utils.js b/packages/ipfs/src/core/components/files/utils/hamt-utils.js index c1bfa24a8d..17466eff73 100644 --- a/packages/ipfs/src/core/components/files/utils/hamt-utils.js +++ b/packages/ipfs/src/core/components/files/utils/hamt-utils.js @@ -10,11 +10,10 @@ const UnixFS = require('ipfs-unixfs') const mc = require('multicodec') const mh = require('multihashing-async').multihash const last = require('it-last') -const { Buffer } = require('buffer') const updateHamtDirectory = async (context, links, bucket, options) => { // update parent with new bit field - const data = Buffer.from(bucket._children.bitField().reverse()) + const data = Uint8Array.from(bucket._children.bitField().reverse()) const node = UnixFS.unmarshal(options.parent.Data) const dir = new UnixFS({ type: 'hamt-sharded-directory', diff --git a/packages/ipfs/src/core/components/files/utils/to-async-iterator.js b/packages/ipfs/src/core/components/files/utils/to-async-iterator.js index c289b2fcab..6ed0f1b837 100644 --- a/packages/ipfs/src/core/components/files/utils/to-async-iterator.js +++ b/packages/ipfs/src/core/components/files/utils/to-async-iterator.js @@ -1,11 +1,11 @@ 'use strict' const errCode = require('err-code') -const { Buffer } = require('buffer') const log = require('debug')('ipfs:mfs:utils:to-async-iterator') const { MFS_MAX_CHUNK_SIZE } = require('../../../utils') +const uint8ArrayFromString = require('uint8arrays/from-string') const toAsyncIterator = (content) => { if (!content) { @@ -15,7 +15,7 @@ const toAsyncIterator = (content) => { if (typeof content === 'string' || content instanceof String) { log('Content was a string') - content = Buffer.from(content) + content = uint8ArrayFromString(content) } if (content.length) { @@ -67,7 +67,7 @@ const toAsyncIterator = (content) => { resolve({ done: false, - value: Buffer.from(reader.result) + value: new Uint8Array(reader.result, reader.result.byteOffset, reader.result.byteLength) }) } diff --git a/packages/ipfs/src/core/components/files/utils/update-mfs-root.js b/packages/ipfs/src/core/components/files/utils/update-mfs-root.js index e936229bea..d172c34c76 100644 --- a/packages/ipfs/src/core/components/files/utils/update-mfs-root.js +++ b/packages/ipfs/src/core/components/files/utils/update-mfs-root.js @@ -13,7 +13,7 @@ const updateMfsRoot = async (context, cid, options) => { log(`New MFS root will be ${cid}`) - await context.repo.datastore.put(MFS_ROOT_KEY, cid.buffer) + await context.repo.datastore.put(MFS_ROOT_KEY, cid.bytes) return cid } diff --git a/packages/ipfs/src/core/components/files/utils/with-mfs-root.js b/packages/ipfs/src/core/components/files/utils/with-mfs-root.js index 6c1f7f1bcb..a25b0ceb73 100644 --- a/packages/ipfs/src/core/components/files/utils/with-mfs-root.js +++ b/packages/ipfs/src/core/components/files/utils/with-mfs-root.js @@ -45,7 +45,7 @@ const loadMfsRoot = async (context, options) => { throw errCode(new Error('Request aborted'), 'ERR_ABORTED', { name: 'Aborted' }) } - await context.repo.datastore.put(MFS_ROOT_KEY, cid.buffer) + await context.repo.datastore.put(MFS_ROOT_KEY, cid.bytes) } log(`Loaded MFS root /ipfs/${cid}`) diff --git a/packages/ipfs/src/core/components/files/write.js b/packages/ipfs/src/core/components/files/write.js index e524126993..a768cba977 100644 --- a/packages/ipfs/src/core/components/files/write.js +++ b/packages/ipfs/src/core/components/files/write.js @@ -2,7 +2,6 @@ const log = require('debug')('ipfs:mfs:write') const importer = require('ipfs-unixfs-importer') -const { Buffer } = require('buffer') const stat = require('./stat') const mkdir = require('./mkdir') const addLink = require('./utils/add-link') @@ -245,7 +244,7 @@ const limitAsyncStreamBytes = (stream, limit) => { } const asyncZeroes = (count, chunkSize = MFS_MAX_CHUNK_SIZE) => { - const buf = Buffer.alloc(chunkSize, 0) + const buf = new Uint8Array(chunkSize, 0) const stream = { [Symbol.asyncIterator]: function * _asyncZeroes () { diff --git a/packages/ipfs/src/core/components/id.js b/packages/ipfs/src/core/components/id.js index 49953ef5cb..5e911fbd6d 100644 --- a/packages/ipfs/src/core/components/id.js +++ b/packages/ipfs/src/core/components/id.js @@ -3,6 +3,7 @@ const pkgversion = require('../../../package.json').version const multiaddr = require('multiaddr') const { withTimeoutOption } = require('../utils') +const uint8ArrayToString = require('uint8arrays/to-string') module.exports = ({ peerId, libp2p }) => { return withTimeoutOption(async function id () { // eslint-disable-line require-await @@ -16,7 +17,7 @@ module.exports = ({ peerId, libp2p }) => { return { id, - publicKey: peerId.pubKey.bytes.toString('base64'), + publicKey: uint8ArrayToString(peerId.pubKey.bytes, 'base64pad'), addresses: addresses .map(ma => { const str = ma.toString() diff --git a/packages/ipfs/src/core/components/init.js b/packages/ipfs/src/core/components/init.js index b54de2544d..e6adadb49c 100644 --- a/packages/ipfs/src/core/components/init.js +++ b/packages/ipfs/src/core/components/init.js @@ -2,7 +2,8 @@ const log = require('debug')('ipfs:components:init') const PeerId = require('peer-id') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require() const mergeOptions = require('merge-options') const getDefaultConfig = require('../runtime/config-nodejs.js') @@ -197,7 +198,7 @@ async function initNewRepo (repo, { privateKey, emptyRepo, algorithm, bits, prof config.Identity = { PeerID: peerId.toB58String(), - PrivKey: peerId.privKey.bytes.toString('base64') + PrivKey: uint8ArrayToString(peerId.privKey.bytes, 'base64pad') } privateKey = peerId.privKey @@ -265,7 +266,7 @@ function createPeerId ({ privateKey, algorithm = 'rsa', bits, print }) { log('using user-supplied private-key') return typeof privateKey === 'object' ? privateKey - : PeerId.createFromPrivKey(Buffer.from(privateKey, 'base64')) + : PeerId.createFromPrivKey(uint8ArrayFromString(privateKey, 'base64')) } else { // Generate peer identity keypair + transform to desired format + add to config. print('generating %s-bit (rsa only) %s keypair...', bits, algorithm) diff --git a/packages/ipfs/src/core/components/object/get.js b/packages/ipfs/src/core/components/object/get.js index d32ecffefe..8d96a18e24 100644 --- a/packages/ipfs/src/core/components/object/get.js +++ b/packages/ipfs/src/core/components/object/get.js @@ -2,19 +2,19 @@ const CID = require('cids') const errCode = require('err-code') -const { Buffer } = require('buffer') const { withTimeoutOption } = require('../../utils') +const uint8ArrayFromString = require('uint8arrays/from-string') function normalizeMultihash (multihash, enc) { if (typeof multihash === 'string') { if (enc === 'base58' || !enc) { return multihash } - return Buffer.from(multihash, enc) - } else if (Buffer.isBuffer(multihash)) { + return uint8ArrayFromString(multihash, enc) + } else if (multihash instanceof Uint8Array) { return multihash } else if (CID.isCID(multihash)) { - return multihash.buffer + return multihash.bytes } throw new Error('unsupported multihash') } diff --git a/packages/ipfs/src/core/components/object/new.js b/packages/ipfs/src/core/components/object/new.js index 4c0ebaedee..f7e24fdbbc 100644 --- a/packages/ipfs/src/core/components/object/new.js +++ b/packages/ipfs/src/core/components/object/new.js @@ -5,7 +5,6 @@ const DAGNode = dagPB.DAGNode const multicodec = require('multicodec') const Unixfs = require('ipfs-unixfs') const { withTimeoutOption } = require('../../utils') -const { Buffer } = require('buffer') module.exports = ({ ipld, preload }) => { return withTimeoutOption(async function _new (options = {}) { @@ -18,7 +17,7 @@ module.exports = ({ ipld, preload }) => { throw new Error('unknown template') } } else { - data = Buffer.alloc(0) + data = new Uint8Array(0) } const node = new DAGNode(data) diff --git a/packages/ipfs/src/core/components/object/patch/append-data.js b/packages/ipfs/src/core/components/object/patch/append-data.js index ed02a97936..df31ca668c 100644 --- a/packages/ipfs/src/core/components/object/patch/append-data.js +++ b/packages/ipfs/src/core/components/object/patch/append-data.js @@ -2,7 +2,7 @@ const { DAGNode } = require('ipld-dag-pb') const { withTimeoutOption } = require('../../../utils') -const { Buffer } = require('buffer') +const uint8ArrayConcat = require('uint8arrays/concat') module.exports = ({ ipld, gcLock, preload }) => { const get = require('../get')({ ipld, preload }) @@ -10,7 +10,7 @@ module.exports = ({ ipld, gcLock, preload }) => { return withTimeoutOption(async function appendData (multihash, data, options) { const node = await get(multihash, options) - const newData = Buffer.concat([node.Data, data]) + const newData = uint8ArrayConcat([node.Data, data]) return put(new DAGNode(newData, node.Links), options) }) } diff --git a/packages/ipfs/src/core/components/object/put.js b/packages/ipfs/src/core/components/object/put.js index 6b80207685..48b901b13a 100644 --- a/packages/ipfs/src/core/components/object/put.js +++ b/packages/ipfs/src/core/components/object/put.js @@ -6,7 +6,8 @@ const DAGLink = dagPB.DAGLink const mh = require('multihashing-async').multihash const multicodec = require('multicodec') const { withTimeoutOption } = require('../../utils') -const { Buffer } = require('buffer') +const uint8ArrayToString = require('uint8arrays/to-string') +const uint8ArrayFromString = require('uint8arrays/from-string') function parseBuffer (buf, encoding) { switch (encoding) { @@ -24,7 +25,7 @@ function parseJSONBuffer (buf) { let links try { - const parsed = JSON.parse(buf.toString()) + const parsed = JSON.parse(uint8ArrayToString(buf)) links = (parsed.Links || []).map((link) => { return new DAGLink( @@ -33,7 +34,7 @@ function parseJSONBuffer (buf) { mh.fromB58String(link.Hash || link.hash || link.multihash) ) }) - data = Buffer.from(parsed.Data) + data = uint8ArrayFromString(parsed.Data) } catch (err) { throw new Error('failed to parse JSON: ' + err) } @@ -52,7 +53,7 @@ module.exports = ({ ipld, gcLock, preload }) => { const encoding = options.enc let node - if (Buffer.isBuffer(obj)) { + if (obj instanceof Uint8Array) { if (encoding) { node = await parseBuffer(obj, encoding) } else { diff --git a/packages/ipfs/src/core/components/pin/pin-manager.js b/packages/ipfs/src/core/components/pin/pin-manager.js index 633dd653f2..84112e63ab 100644 --- a/packages/ipfs/src/core/components/pin/pin-manager.js +++ b/packages/ipfs/src/core/components/pin/pin-manager.js @@ -22,12 +22,14 @@ function invalidPinTypeErr (type) { return errCode(new Error(errMsg), 'ERR_INVALID_PIN_TYPE') } +const encoder = multibase.encoding('base32upper') + function cidToKey (cid) { - return `/${multibase.encode('base32upper', cid.multihash).toString()}` + return `/${encoder.encode(cid.multihash)}` } function keyToMultihash (key) { - return multibase.decode(`${key.toString().slice(1)}`) + return encoder.decode(key.toString().slice(1)) } const PinTypes = { diff --git a/packages/ipfs/src/core/index.js b/packages/ipfs/src/core/index.js index 4f0c79d1f5..d13df9ed85 100644 --- a/packages/ipfs/src/core/index.js +++ b/packages/ipfs/src/core/index.js @@ -5,7 +5,6 @@ const mergeOptions = require('merge-options') const { isTest } = require('ipfs-utils/src/env') const globSource = require('ipfs-utils/src/files/glob-source') const urlSource = require('ipfs-utils/src/files/url-source') -const { Buffer } = require('buffer') const PeerId = require('peer-id') const crypto = require('libp2p-crypto') const isIPFS = require('is-ipfs') diff --git a/packages/ipfs/src/core/ipns/publisher.js b/packages/ipfs/src/core/ipns/publisher.js index 393126516a..4fccf589ab 100644 --- a/packages/ipfs/src/core/ipns/publisher.js +++ b/packages/ipfs/src/core/ipns/publisher.js @@ -6,6 +6,7 @@ const errcode = require('err-code') const debug = require('debug') const log = debug('ipfs:ipns:publisher') log.error = debug('ipfs:ipns:publisher:error') +const uint8ArrayToString = require('uint8arrays/to-string') const ipns = require('ipns') @@ -79,12 +80,12 @@ class IpnsPublisher { // Add record to routing (buffer key) try { - const res = await this._routing.put(key.toBuffer(), entryData) - log(`ipns record for ${key.toString('base64')} was stored in the routing`) + const res = await this._routing.put(key.uint8Array(), entryData) + log(`ipns record for ${uint8ArrayToString(key.uint8Array(), 'base64')} was stored in the routing`) return res } catch (err) { - const errMsg = `ipns record for ${key.toString('base64')} could not be stored in the routing` + const errMsg = `ipns record for ${uint8ArrayToString(key.uint8Array(), 'base64')} could not be stored in the routing` log.error(errMsg) log.error(err) @@ -109,12 +110,12 @@ class IpnsPublisher { // Add public key to routing (buffer key) try { - const res = await this._routing.put(key.toBuffer(), publicKey.bytes) - log(`public key for ${key.toString('base64')} was stored in the routing`) + const res = await this._routing.put(key.uint8Array(), publicKey.bytes) + log(`public key for ${uint8ArrayToString(key.uint8Array(), 'base64')} was stored in the routing`) return res } catch (err) { - const errMsg = `public key for ${key.toString('base64')} could not be stored in the routing` + const errMsg = `public key for ${uint8ArrayToString(key.uint8Array(), 'base64')} could not be stored in the routing` log.error(errMsg) log.error(err) @@ -156,7 +157,7 @@ class IpnsPublisher { // Try to get from routing try { const keys = ipns.getIdKeys(peerId.toBytes()) - const res = await this._routing.get(keys.routingKey.toBuffer()) + const res = await this._routing.get(keys.routingKey.uint8Array()) // unmarshal data return this._unmarshalData(res) @@ -194,7 +195,7 @@ class IpnsPublisher { record = await this._getPublished(peerId, getPublishedOptions) } catch (err) { if (err.code !== ERR_NOT_FOUND) { - const errMsg = `unexpected error when determining the last published IPNS record for ${peerId.id}` + const errMsg = `unexpected error when determining the last published IPNS record for ${peerId.id} ${err.stack}` log.error(errMsg) throw errcode(new Error(errMsg), 'ERR_DETERMINING_PUBLISHED_RECORD') diff --git a/packages/ipfs/src/core/ipns/republisher.js b/packages/ipfs/src/core/ipns/republisher.js index b5b196c506..78bdd5edbb 100644 --- a/packages/ipfs/src/core/ipns/republisher.js +++ b/packages/ipfs/src/core/ipns/republisher.js @@ -4,7 +4,6 @@ const ipns = require('ipns') const crypto = require('libp2p-crypto') const PeerId = require('peer-id') const errcode = require('err-code') -const { Buffer } = require('buffer') const debug = require('debug') const log = debug('ipfs:ipns:republisher') @@ -148,7 +147,7 @@ class IpnsRepublisher { try { const dsVal = await this._datastore.get(ipns.getLocalKey(peerId.id)) - if (!Buffer.isBuffer(dsVal)) { + if (!(dsVal instanceof Uint8Array)) { throw errcode(new Error("found ipns record that we couldn't process"), 'ERR_INVALID_IPNS_RECORD') } diff --git a/packages/ipfs/src/core/ipns/resolver.js b/packages/ipfs/src/core/ipns/resolver.js index 57bf65e740..cb884d75f2 100644 --- a/packages/ipfs/src/core/ipns/resolver.js +++ b/packages/ipfs/src/core/ipns/resolver.js @@ -78,15 +78,15 @@ class IpnsResolver { let record try { - record = await this._routing.get(routingKey.toBuffer()) + record = await this._routing.get(routingKey.uint8Array()) } catch (err) { - log.error(err) + log.error('could not get record from routing', err) if (err.code === ERR_NOT_FOUND) { throw errcode(new Error(`record requested for ${name} was not found in the network`), 'ERR_NO_RECORD_FOUND') } - throw errcode(new Error(`unexpected error getting the ipns record ${peerId.id}`), 'ERR_UNEXPECTED_ERROR_GETTING_RECORD') + throw errcode(new Error(`unexpected error getting the ipns record ${peerId.toString()}`), 'ERR_UNEXPECTED_ERROR_GETTING_RECORD') } // IPNS entry @@ -94,7 +94,7 @@ class IpnsResolver { try { ipnsEntry = ipns.unmarshal(record) } catch (err) { - log.error(err) + log.error('could not unmarshal record', err) throw errcode(new Error('found ipns record that we couldn\'t convert to a value'), 'ERR_INVALID_RECORD_RECEIVED') } @@ -107,9 +107,9 @@ class IpnsResolver { // Otherwise, try to get the public key from routing let pubKey try { - pubKey = await this._routing.get(routingKey.toBuffer()) + pubKey = await this._routing.get(routingKey.uint8Array()) } catch (err) { - log.error(err) + log.error('could not get public key for routing key', err) if (err.code === ERR_NOT_FOUND) { throw errcode(new Error(`public key requested for ${name} was not found in the network`), 'ERR_NO_RECORD_FOUND') @@ -122,7 +122,7 @@ class IpnsResolver { // Insert it into the peer id, in order to be validated by IPNS validator peerId.pubKey = crypto.keys.unmarshalPublicKey(pubKey) } catch (err) { - log.error(err) + log.error('could not unmarshal public key', err) throw errcode(new Error('found public key record that we couldn\'t convert to a value'), 'ERR_INVALID_PUB_KEY_RECEIVED') } diff --git a/packages/ipfs/src/core/ipns/routing/offline-datastore.js b/packages/ipfs/src/core/ipns/routing/offline-datastore.js index bd9e748217..b20780018c 100644 --- a/packages/ipfs/src/core/ipns/routing/offline-datastore.js +++ b/packages/ipfs/src/core/ipns/routing/offline-datastore.js @@ -2,7 +2,6 @@ const { Key } = require('interface-datastore') const { Record } = require('libp2p-record') -const { Buffer } = require('buffer') const { encodeBase32 } = require('./utils') const errcode = require('err-code') @@ -19,18 +18,18 @@ class OfflineDatastore { /** * Put a value to the local datastore indexed by the received key properly encoded. - * @param {Buffer} key identifier of the value. - * @param {Buffer} value value to be stored. + * @param {Uint8Array} key identifier of the value. + * @param {Uint8Array} value value to be stored. * @param {function(Error)} callback * @returns {void} */ async put (key, value) { // eslint-disable-line require-await - if (!Buffer.isBuffer(key)) { - throw errcode(new Error('Offline datastore key must be a buffer'), 'ERR_INVALID_KEY') + if (!(key instanceof Uint8Array)) { + throw errcode(new Error('Offline datastore key must be a Uint8Array'), 'ERR_INVALID_KEY') } - if (!Buffer.isBuffer(value)) { - throw errcode(new Error('Offline datastore value must be a buffer'), 'ERR_INVALID_VALUE') + if (!(value instanceof Uint8Array)) { + throw errcode(new Error('Offline datastore value must be a Uint8Array'), 'ERR_INVALID_VALUE') } let routingKey @@ -50,13 +49,13 @@ class OfflineDatastore { /** * Get a value from the local datastore indexed by the received key properly encoded. - * @param {Buffer} key identifier of the value to be obtained. - * @param {function(Error, Buffer)} callback + * @param {Uint8Array} key identifier of the value to be obtained. + * @param {function(Error, Uint8Array)} callback * @returns {void} */ async get (key) { - if (!Buffer.isBuffer(key)) { - throw errcode(new Error('Offline datastore key must be a buffer'), 'ERR_INVALID_KEY') + if (!(key instanceof Uint8Array)) { + throw errcode(new Error('Offline datastore key must be a Uint8Array'), 'ERR_INVALID_KEY') } let routingKey @@ -76,7 +75,7 @@ class OfflineDatastore { record = Record.deserialize(res) } catch (err) { log.error(err) - throw (err) + throw err } return record.value diff --git a/packages/ipfs/src/core/ipns/routing/pubsub-datastore.js b/packages/ipfs/src/core/ipns/routing/pubsub-datastore.js index 52bb8df8e1..796ef40b73 100644 --- a/packages/ipfs/src/core/ipns/routing/pubsub-datastore.js +++ b/packages/ipfs/src/core/ipns/routing/pubsub-datastore.js @@ -1,9 +1,10 @@ 'use strict' const ipns = require('ipns') -const { Buffer } = require('buffer') -const { fromB58String, toB58String } = require('multihashing-async').multihash +const { toB58String } = require('multihashing-async').multihash const PubsubDatastore = require('datastore-pubsub') +const uint8ArrayToString = require('uint8arrays/to-string') +const uint8ArrayFromString = require('uint8arrays/from-string') const withIs = require('class-is') @@ -55,7 +56,7 @@ class IpnsPubsubDatastore { // Add topic subscribed const ns = key.slice(0, ipns.namespaceLength) - if (ns.toString() === ipns.namespace) { + if (uint8ArrayToString(ns) === ipns.namespace) { const stringifiedTopic = toB58String(key) const id = toB58String(key.slice(ipns.namespaceLength)) @@ -74,8 +75,8 @@ class IpnsPubsubDatastore { // Modify subscription key to have a proper encoding _handleSubscriptionKey (key) { - if (Buffer.isBuffer(key)) { - key = toB58String(key) + if (key instanceof Uint8Array) { + key = uint8ArrayToString(key, 'base58btc') } const subscriber = this._subscriptions[key] @@ -86,13 +87,13 @@ class IpnsPubsubDatastore { let keys try { - keys = ipns.getIdKeys(fromB58String(subscriber)) + keys = ipns.getIdKeys(uint8ArrayFromString(subscriber, 'base58btc')) } catch (err) { log.error(err) throw err } - return keys.routingKey.toBuffer() + return keys.routingKey.uint8Array() } /** @@ -132,7 +133,7 @@ class IpnsPubsubDatastore { } // Unsubscribe topic - const bufTopic = Buffer.from(stringifiedTopic) + const bufTopic = uint8ArrayFromString(stringifiedTopic) this._pubsubDs.unsubscribe(bufTopic) diff --git a/packages/ipfs/src/core/ipns/routing/utils.js b/packages/ipfs/src/core/ipns/routing/utils.js index 0d960618ce..e6c8e7841c 100644 --- a/packages/ipfs/src/core/ipns/routing/utils.js +++ b/packages/ipfs/src/core/ipns/routing/utils.js @@ -1,14 +1,10 @@ 'use strict' -const multibase = require('multibase') const ipns = require('ipns') +const uint8ArrayToString = require('uint8arrays/to-string') module.exports = { - encodeBase32: (buf) => { - const m = multibase.encode('base32', buf).slice(1) // slice off multibase codec - - return m.toString().toUpperCase() // should be uppercase for interop with go - }, + encodeBase32: (buf) => uint8ArrayToString(buf, 'base32upper'), validator: { func: (key, record, cb) => ipns.validator.validate(record, key, cb) }, diff --git a/packages/ipfs/src/core/utils.js b/packages/ipfs/src/core/utils.js index 01ea555fee..2fd8594b37 100644 --- a/packages/ipfs/src/core/utils.js +++ b/packages/ipfs/src/core/utils.js @@ -2,7 +2,6 @@ const isIpfs = require('is-ipfs') const CID = require('cids') -const { Buffer } = require('buffer') const TimeoutController = require('timeout-abort-controller') const anySignal = require('any-signal') const parseDuration = require('parse-duration').default @@ -44,7 +43,7 @@ const normalizePath = (pathStr) => { // TODO: do we need both normalizePath and normalizeCidPath? const normalizeCidPath = (path) => { - if (Buffer.isBuffer(path)) { + if (path instanceof Uint8Array) { return new CID(path).toString() } if (CID.isCID(path)) { diff --git a/packages/ipfs/src/http/api/resources/block.js b/packages/ipfs/src/http/api/resources/block.js index 3dc3dad337..a7d1b03268 100644 --- a/packages/ipfs/src/http/api/resources/block.js +++ b/packages/ipfs/src/http/api/resources/block.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const multihash = require('multihashing-async').multihash const codecs = require('multicodec/src/base-table.json') const multipart = require('../../utils/multipart-request-parser') diff --git a/packages/ipfs/src/http/api/resources/config.js b/packages/ipfs/src/http/api/resources/config.js index bc54d47bd6..5cecd021d4 100644 --- a/packages/ipfs/src/http/api/resources/config.js +++ b/packages/ipfs/src/http/api/resources/config.js @@ -1,7 +1,6 @@ 'use strict' const debug = require('debug') -const { Buffer } = require('buffer') const get = require('dlv') const set = require('just-safe-set') const log = debug('ipfs:http-api:config') diff --git a/packages/ipfs/src/http/api/resources/dag.js b/packages/ipfs/src/http/api/resources/dag.js index 63df10932c..5eedee01c5 100644 --- a/packages/ipfs/src/http/api/resources/dag.js +++ b/packages/ipfs/src/http/api/resources/dag.js @@ -6,13 +6,13 @@ const Joi = require('../../utils/joi') const multicodec = require('multicodec') const Boom = require('@hapi/boom') const debug = require('debug') -const { Buffer } = require('buffer') const { cidToString } = require('../../../utils/cid') const all = require('it-all') const log = debug('ipfs:http-api:dag') log.error = debug('ipfs:http-api:dag:error') +const uint8ArrayToString = require('uint8arrays/to-string') const IpldFormats = { get [multicodec.RAW] () { @@ -61,13 +61,13 @@ const encodeBufferKeys = (obj, encoding) => { return obj } - if (Buffer.isBuffer(obj)) { - return obj.toString(encoding) + if (obj instanceof Uint8Array) { + return uint8ArrayToString(obj, encoding) } Object.keys(obj).forEach(key => { - if (Buffer.isBuffer(obj)) { - obj[key] = obj[key].toString(encoding) + if (obj instanceof Uint8Array) { + obj[key] = uint8ArrayToString(obj[key], encoding) return } @@ -90,8 +90,10 @@ exports.get = { query: Joi.object().keys({ arg: Joi.cidAndPath().required(), dataEncoding: Joi.string() - .valid('text', 'base64', 'hex', 'utf8') - .replace(/text/, 'utf8') + .valid('ascii', 'base64pad', 'base16', 'utf8') + .replace(/text/, 'ascii') + .replace(/base64/, 'base64pad') + .replace(/hex/, 'base16') .default('utf8'), timeout: Joi.timeout() }) @@ -135,7 +137,7 @@ exports.get = { let value = result.value - if (!Buffer.isBuffer(result.value) && result.value.toJSON) { + if (!(result.value instanceof Uint8Array) && result.value.toJSON) { value = result.value.toJSON() } diff --git a/packages/ipfs/src/http/api/resources/object.js b/packages/ipfs/src/http/api/resources/object.js index 4b2067234d..98904bc749 100644 --- a/packages/ipfs/src/http/api/resources/object.js +++ b/packages/ipfs/src/http/api/resources/object.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const multipart = require('../../utils/multipart-request-parser') const all = require('it-all') const dagPB = require('ipld-dag-pb') diff --git a/packages/ipfs/src/http/gateway/resources/gateway.js b/packages/ipfs/src/http/gateway/resources/gateway.js index 67c4a787d4..986d269032 100644 --- a/packages/ipfs/src/http/gateway/resources/gateway.js +++ b/packages/ipfs/src/http/gateway/resources/gateway.js @@ -1,7 +1,8 @@ 'use strict' const debug = require('debug') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require('uint8arrays/to-string') const Boom = require('@hapi/boom') const Ammo = require('@hapi/ammo') // HTTP Range processing utilities const last = require('it-last') @@ -192,7 +193,7 @@ module.exports = { // encode CID/FQDN in base32 (Suborigin allows only a-z) const ipnsOrigin = isIPFS.cid(root) ? cidToString(root, { base: 'base32' }) - : multibase.encode('base32', Buffer.from(root)).toString() + : uint8ArrayToString(multibase.encode('base32', uint8ArrayFromString(root))) response.header('Suborigin', `ipns000${ipnsOrigin}`) } } diff --git a/packages/ipfs/src/http/utils/multipart-request-parser.js b/packages/ipfs/src/http/utils/multipart-request-parser.js index 84ea04be69..f621cf85c8 100644 --- a/packages/ipfs/src/http/utils/multipart-request-parser.js +++ b/packages/ipfs/src/http/utils/multipart-request-parser.js @@ -2,7 +2,8 @@ const Content = require('@hapi/content') const multipart = require('it-multipart') -const { Buffer } = require('buffer') +const uint8ArrayConcat = require('uint8arrays/concat') +const uint8ArrayToString = require('uint8arrays/to-string') const qs = require('querystring') const multipartFormdataType = 'multipart/form-data' @@ -37,7 +38,7 @@ const collect = async (stream) => { buffers.push(buf) } - return Buffer.concat(buffers, size) + return uint8ArrayConcat(buffers, size) } const ignore = async (stream) => { @@ -119,7 +120,7 @@ async function * parser (stream, options) { yield { type: 'symlink', name: entry.name, - target: (await collect(entry.body)).toString('utf8'), + target: uint8ArrayToString(await collect(entry.body)), mtime: entry.mtime, mode: entry.mode } diff --git a/packages/ipfs/test/cli/add.js b/packages/ipfs/test/cli/add.js index 459f59fefc..13f7e914df 100644 --- a/packages/ipfs/test/cli/add.js +++ b/packages/ipfs/test/cli/add.js @@ -1,12 +1,12 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const CID = require('cids') const first = require('it-first') const cli = require('../utils/cli') const sinon = require('sinon') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') // TODO: Test against all algorithms Object.keys(mh.names) // This subset is known to work with both go-ipfs and js-ipfs as of 2017-09-05 @@ -156,7 +156,7 @@ describe('add', () => { const proc = cli('add', { ipfs, getStdin: function * () { - yield Buffer.from('hello\n') + yield uint8ArrayFromString('hello\n') } }) @@ -178,7 +178,7 @@ describe('add', () => { const proc = cli('add --mtime=100', { ipfs, getStdin: function * () { - yield Buffer.from('hello\n') + yield uint8ArrayFromString('hello\n') } }) diff --git a/packages/ipfs/test/cli/bitswap.js b/packages/ipfs/test/cli/bitswap.js index e8b48bd630..f03169631b 100644 --- a/packages/ipfs/test/cli/bitswap.js +++ b/packages/ipfs/test/cli/bitswap.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const CID = require('cids') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/block.js b/packages/ipfs/test/cli/block.js index 66906fd719..958790484a 100644 --- a/packages/ipfs/test/cli/block.js +++ b/packages/ipfs/test/cli/block.js @@ -1,11 +1,11 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const CID = require('cids') const cli = require('../utils/cli') const sinon = require('sinon') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') describe('block', () => { const cid = new CID('QmZjTnYw2TFhn9Nn7tjmPSoTBoY7YRkwPzwSrSbabY24Kp') @@ -96,7 +96,7 @@ describe('block', () => { it('should get a block', async () => { ipfs.block.get.withArgs(cid.toString(), defaultOptions).resolves({ cid, - data: Buffer.from('hello world\n') + data: uint8ArrayFromString('hello world\n') }) const out = await cli(`block get ${cid}`, { ipfs }) @@ -114,7 +114,7 @@ describe('block', () => { timeout: 1000 }).resolves({ cid, - data: Buffer.from('hello world\n') + data: uint8ArrayFromString('hello world\n') }) const out = await cli(`block get ${cid} --timeout=1s`, { ipfs }) diff --git a/packages/ipfs/test/cli/bootstrap.js b/packages/ipfs/test/cli/bootstrap.js index 0d1ba24951..888b8a286a 100644 --- a/packages/ipfs/test/cli/bootstrap.js +++ b/packages/ipfs/test/cli/bootstrap.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/cat.js b/packages/ipfs/test/cli/cat.js index 7faac5b4e2..c91875b0b4 100644 --- a/packages/ipfs/test/cli/cat.js +++ b/packages/ipfs/test/cli/cat.js @@ -1,10 +1,12 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const CID = require('cids') const cli = require('../utils/cli') const sinon = require('sinon') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require('uint8arrays/to-string') const defaultOptions = { offset: undefined, @@ -23,17 +25,17 @@ describe('cat', () => { it('should cat a file', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - const buf = Buffer.from('hello world') + const buf = uint8ArrayFromString('hello world') ipfs.cat.withArgs(cid.toString(), defaultOptions).returns([buf]) const out = await cli(`cat ${cid}`, { ipfs }) - expect(out).to.equal(buf.toString('utf8')) + expect(out).to.equal(uint8ArrayToString(buf)) }) it('cat part of a file using `count`', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - const buf = Buffer.from('hello world') + const buf = uint8ArrayFromString('hello world') ipfs.cat.withArgs(cid.toString(), { ...defaultOptions, @@ -42,12 +44,12 @@ describe('cat', () => { }).returns([buf]) const out = await cli('cat QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB --offset 21 --count 5', { ipfs }) - expect(out).to.equal(buf.toString('utf8')) + expect(out).to.equal(uint8ArrayToString(buf)) }) it('cat part of a file using `length`', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - const buf = Buffer.from('hello world') + const buf = uint8ArrayFromString('hello world') ipfs.cat.withArgs(cid.toString(), { ...defaultOptions, @@ -56,7 +58,7 @@ describe('cat', () => { }).returns([buf]) const out = await cli('cat QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB --offset 21 --length 5', { ipfs }) - expect(out).to.equal(buf.toString('utf8')) + expect(out).to.equal(uint8ArrayToString(buf)) }) it('cat non-existent file', async () => { @@ -71,7 +73,7 @@ describe('cat', () => { it('should cat a file with a timeout', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - const buf = Buffer.from('hello world') + const buf = uint8ArrayFromString('hello world') ipfs.cat.withArgs(cid.toString(), { ...defaultOptions, @@ -79,6 +81,6 @@ describe('cat', () => { }).returns([buf]) const out = await cli(`cat ${cid} --timeout=1s`, { ipfs }) - expect(out).to.equal(buf.toString('utf8')) + expect(out).to.equal(uint8ArrayToString(buf)) }) }) diff --git a/packages/ipfs/test/cli/commands.js b/packages/ipfs/test/cli/commands.js index 8bd80ae3a8..473e5fe40b 100644 --- a/packages/ipfs/test/cli/commands.js +++ b/packages/ipfs/test/cli/commands.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const commandCount = 110 diff --git a/packages/ipfs/test/cli/config.js b/packages/ipfs/test/cli/config.js index cc9dd9d258..ae39d6aac3 100644 --- a/packages/ipfs/test/cli/config.js +++ b/packages/ipfs/test/cli/config.js @@ -2,7 +2,7 @@ /* eslint max-nested-callbacks: ["error", 5] */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') const { profiles } = require('../../src/core/components/config') diff --git a/packages/ipfs/test/cli/daemon.js b/packages/ipfs/test/cli/daemon.js index ad59f82b27..040073f8c3 100644 --- a/packages/ipfs/test/cli/daemon.js +++ b/packages/ipfs/test/cli/daemon.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const clean = require('../utils/clean') const ipfsCmd = require('../utils/ipfs-exec') const { isWindows } = require('../utils/platforms') diff --git a/packages/ipfs/test/cli/dag.js b/packages/ipfs/test/cli/dag.js index 7a9694363d..b3dc853928 100644 --- a/packages/ipfs/test/cli/dag.js +++ b/packages/ipfs/test/cli/dag.js @@ -1,13 +1,14 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const dagCBOR = require('ipld-dag-cbor') const dagPB = require('ipld-dag-pb') const sinon = require('sinon') const CID = require('cids') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require('uint8arrays/to-string') describe('dag', () => { const cid = new CID('Qmaj2NmcyAXT8dFmZRRytE12wpcaHADzbChKToMEjBsj5Z') @@ -32,20 +33,20 @@ describe('dag', () => { it('should get a node', async () => { const result = { - value: Buffer.from('hello world') + value: uint8ArrayFromString('hello world') } ipfs.dag.get.withArgs(cid, defaultOptions).returns(result) const out = await cli(`dag get ${cid}`, { ipfs }) - expect(out).to.be.eql('0x' + result.value.toString('hex') + '\n') + expect(out).to.be.eql('0x' + uint8ArrayToString(result.value, 'base16') + '\n') }) it('should get a node with a deep path', async () => { const path = '/parentHash' const result = { - value: Buffer.from('hello world') + value: uint8ArrayFromString('hello world') } ipfs.dag.get.withArgs(cid, { @@ -55,13 +56,13 @@ describe('dag', () => { const out = await cli(`dag get ${cid}${path}`, { ipfs }) - expect(out).to.be.eql('0x' + result.value.toString('hex') + '\n') + expect(out).to.be.eql('0x' + uint8ArrayToString(result.value, 'base16') + '\n') }) it('should get a node with a deep path and an ipfs prefix', async () => { const path = '/parentHash' const result = { - value: Buffer.from('hello world') + value: uint8ArrayFromString('hello world') } ipfs.dag.get.withArgs(cid, { @@ -71,12 +72,12 @@ describe('dag', () => { const out = await cli(`dag get /ipfs/${cid}${path}`, { ipfs }) - expect(out).to.be.eql('0x' + result.value.toString('hex') + '\n') + expect(out).to.be.eql('0x' + uint8ArrayToString(result.value, 'base16') + '\n') }) it('should get a node with local resolve', async () => { const result = { - value: Buffer.from('hello world') + value: uint8ArrayFromString('hello world') } ipfs.dag.get.withArgs(cid, { @@ -88,12 +89,12 @@ describe('dag', () => { expect(out).to.include('resolving path within the node only\n') expect(out).to.include('remainder path: n/a\n') - expect(out).to.include('0x' + result.value.toString('hex') + '\n') + expect(out).to.include('0x' + uint8ArrayToString(result.value, 'base16') + '\n') }) it('should get a node with a timeout', async () => { const result = { - value: Buffer.from('hello world') + value: uint8ArrayFromString('hello world') } ipfs.dag.get.withArgs(cid, { @@ -103,7 +104,7 @@ describe('dag', () => { const out = await cli(`dag get ${cid} --timeout=1s`, { ipfs }) - expect(out).to.be.eql('0x' + result.value.toString('hex') + '\n') + expect(out).to.be.eql('0x' + uint8ArrayToString(result.value, 'base16') + '\n') }) }) @@ -166,7 +167,7 @@ describe('dag', () => { const out = await cli('dag put', { getStdin: function * () { - yield Buffer.from('{}') + yield uint8ArrayFromString('{}') }, ipfs }) diff --git a/packages/ipfs/test/cli/dht.js b/packages/ipfs/test/cli/dht.js index 49423a1b1a..0c0a6e3715 100644 --- a/packages/ipfs/test/cli/dht.js +++ b/packages/ipfs/test/cli/dht.js @@ -2,7 +2,7 @@ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/dns.js b/packages/ipfs/test/cli/dns.js index 6bd6a9a736..27384e63ed 100644 --- a/packages/ipfs/test/cli/dns.js +++ b/packages/ipfs/test/cli/dns.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/files/chmod.js b/packages/ipfs/test/cli/files/chmod.js index 165bd20376..1297fb6d92 100644 --- a/packages/ipfs/test/cli/files/chmod.js +++ b/packages/ipfs/test/cli/files/chmod.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/files/cp.js b/packages/ipfs/test/cli/files/cp.js index 3ef4fa6ab2..7b14cb41d6 100644 --- a/packages/ipfs/test/cli/files/cp.js +++ b/packages/ipfs/test/cli/files/cp.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/files/flush.js b/packages/ipfs/test/cli/files/flush.js index 9e5bba727e..f6cbea4e25 100644 --- a/packages/ipfs/test/cli/files/flush.js +++ b/packages/ipfs/test/cli/files/flush.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../../utils/cli') const sinon = require('sinon') const CID = require('cids') diff --git a/packages/ipfs/test/cli/files/ls.js b/packages/ipfs/test/cli/files/ls.js index 6ed54704da..d873e61d2d 100644 --- a/packages/ipfs/test/cli/files/ls.js +++ b/packages/ipfs/test/cli/files/ls.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../../utils/cli') const sinon = require('sinon') const { isNode } = require('ipfs-utils/src/env') diff --git a/packages/ipfs/test/cli/files/mkdir.js b/packages/ipfs/test/cli/files/mkdir.js index e90148c830..7477930826 100644 --- a/packages/ipfs/test/cli/files/mkdir.js +++ b/packages/ipfs/test/cli/files/mkdir.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../../utils/cli') const sinon = require('sinon') const { isNode } = require('ipfs-utils/src/env') diff --git a/packages/ipfs/test/cli/files/mv.js b/packages/ipfs/test/cli/files/mv.js index cd08305837..2ae16bd34c 100644 --- a/packages/ipfs/test/cli/files/mv.js +++ b/packages/ipfs/test/cli/files/mv.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../../utils/cli') const sinon = require('sinon') const { isNode } = require('ipfs-utils/src/env') diff --git a/packages/ipfs/test/cli/files/read.js b/packages/ipfs/test/cli/files/read.js index 3c198d05a3..a2a1373f93 100644 --- a/packages/ipfs/test/cli/files/read.js +++ b/packages/ipfs/test/cli/files/read.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../../utils/cli') const sinon = require('sinon') const { isNode } = require('ipfs-utils/src/env') diff --git a/packages/ipfs/test/cli/files/rm.js b/packages/ipfs/test/cli/files/rm.js index 353aa07969..7b42162958 100644 --- a/packages/ipfs/test/cli/files/rm.js +++ b/packages/ipfs/test/cli/files/rm.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../../utils/cli') const sinon = require('sinon') const { isNode } = require('ipfs-utils/src/env') diff --git a/packages/ipfs/test/cli/files/stat.js b/packages/ipfs/test/cli/files/stat.js index 4758b24e68..4a45e947a2 100644 --- a/packages/ipfs/test/cli/files/stat.js +++ b/packages/ipfs/test/cli/files/stat.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../../utils/cli') const sinon = require('sinon') const { isNode } = require('ipfs-utils/src/env') diff --git a/packages/ipfs/test/cli/files/touch.js b/packages/ipfs/test/cli/files/touch.js index 5b46acbfc9..1532fbb54d 100644 --- a/packages/ipfs/test/cli/files/touch.js +++ b/packages/ipfs/test/cli/files/touch.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../../utils/cli') const sinon = require('sinon') const { isNode } = require('ipfs-utils/src/env') diff --git a/packages/ipfs/test/cli/files/write.js b/packages/ipfs/test/cli/files/write.js index 0122b0da8d..068f62715e 100644 --- a/packages/ipfs/test/cli/files/write.js +++ b/packages/ipfs/test/cli/files/write.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../../utils/cli') const sinon = require('sinon') const { isNode } = require('ipfs-utils/src/env') diff --git a/packages/ipfs/test/cli/general.js b/packages/ipfs/test/cli/general.js index 5feae2d923..ce37bc8ea1 100644 --- a/packages/ipfs/test/cli/general.js +++ b/packages/ipfs/test/cli/general.js @@ -5,7 +5,7 @@ const os = require('os') const fs = require('fs').promises const path = require('path') const { nanoid } = require('nanoid') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const { repoVersion } = require('ipfs-repo') const { promisify } = require('util') const ncp = promisify(require('ncp').ncp) @@ -14,6 +14,7 @@ const clean = require('../utils/clean') const { isWindows } = require('../utils/platforms') const cli = require('../utils/cli') const sinon = require('sinon') +const uint8ArrayToString = require('uint8arrays/to-string') describe('general cli options', () => { let ipfs @@ -30,7 +31,7 @@ describe('general cli options', () => { }) }) -describe.only('--migrate', () => { +describe('--migrate', () => { let ipfs, repoPath async function setRepoVersion (version) { @@ -83,7 +84,7 @@ describe.only('--migrate', () => { let killed = false daemon.stdout.on('data', data => { - stdout += data.toString('utf8') + stdout += uint8ArrayToString(data) if (stdout.includes('Daemon is ready') && !killed) { killed = true diff --git a/packages/ipfs/test/cli/get.js b/packages/ipfs/test/cli/get.js index 61a48cb692..a3b33cc901 100644 --- a/packages/ipfs/test/cli/get.js +++ b/packages/ipfs/test/cli/get.js @@ -2,12 +2,13 @@ 'use strict' const fs = require('fs') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const path = require('path') const clean = require('../utils/clean') const CID = require('cids') const cli = require('../utils/cli') const sinon = require('sinon') +const uint8ArrayFromString = require('uint8arrays/from-string') const defaultOptions = { timeout: undefined @@ -15,7 +16,7 @@ const defaultOptions = { describe('get', () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - const buf = Buffer.from('hello world') + const buf = uint8ArrayFromString('hello world') let ipfs beforeEach(() => { diff --git a/packages/ipfs/test/cli/id.js b/packages/ipfs/test/cli/id.js index 1ec25fdc13..9ed2ef4d92 100644 --- a/packages/ipfs/test/cli/id.js +++ b/packages/ipfs/test/cli/id.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/init.js b/packages/ipfs/test/cli/init.js index 601733332c..33e28b5f4e 100644 --- a/packages/ipfs/test/cli/init.js +++ b/packages/ipfs/test/cli/init.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const path = require('path') const fs = require('fs') const PeerId = require('peer-id') @@ -11,13 +11,13 @@ const { nanoid } = require('nanoid') const ipfsExec = require('../utils/ipfs-exec') const os = require('os') const tempWrite = require('temp-write') +const uint8ArrayToString = require('uint8arrays/to-string') describe('init', function () { let repoPath let ipfs - const readme = fs.readFileSync(path.join(process.cwd(), '/src/init-files/init-docs/readme')) - .toString('utf-8') + const readme = uint8ArrayToString(fs.readFileSync(path.join(process.cwd(), '/src/init-files/init-docs/readme'))) const repoExistsSync = (p) => fs.existsSync(path.join(repoPath, p)) diff --git a/packages/ipfs/test/cli/key.js b/packages/ipfs/test/cli/key.js index d95b2280ab..4f33990a17 100644 --- a/packages/ipfs/test/cli/key.js +++ b/packages/ipfs/test/cli/key.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/ls.js b/packages/ipfs/test/cli/ls.js index 026da6c70e..75d0a9ebeb 100644 --- a/packages/ipfs/test/cli/ls.js +++ b/packages/ipfs/test/cli/ls.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') const CID = require('cids') diff --git a/packages/ipfs/test/cli/name-pubsub.js b/packages/ipfs/test/cli/name-pubsub.js index e7c3ad9eb9..2a8743fc65 100644 --- a/packages/ipfs/test/cli/name-pubsub.js +++ b/packages/ipfs/test/cli/name-pubsub.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/name.js b/packages/ipfs/test/cli/name.js index 624e5c43c5..e2707033f4 100644 --- a/packages/ipfs/test/cli/name.js +++ b/packages/ipfs/test/cli/name.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/object.js b/packages/ipfs/test/cli/object.js index a52339df9b..e0659d0568 100644 --- a/packages/ipfs/test/cli/object.js +++ b/packages/ipfs/test/cli/object.js @@ -1,14 +1,14 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const path = require('path') const fs = require('fs') const multibase = require('multibase') const cli = require('../utils/cli') const sinon = require('sinon') const CID = require('cids') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { DAGNode, DAGLink @@ -98,7 +98,7 @@ describe('object', () => { }) it('get with data', async () => { - const node = new DAGNode(Buffer.from('aGVsbG8gd29ybGQK', 'base64')) + const node = new DAGNode(uint8ArrayFromString('aGVsbG8gd29ybGQK', 'base64')) ipfs.object.get.withArgs(cid.toString(), defaultOptions).resolves(node) @@ -109,7 +109,7 @@ describe('object', () => { }) it('get while overriding data-encoding', async () => { - const node = new DAGNode(Buffer.from('hello world')) + const node = new DAGNode(uint8ArrayFromString('hello world')) ipfs.object.get.withArgs(cid.toString(), defaultOptions).resolves(node) @@ -157,7 +157,7 @@ describe('object', () => { } it('should put an object', async () => { - ipfs.object.put.withArgs(sinon.match.instanceOf(Buffer), defaultOptions).resolves(cid) + ipfs.object.put.withArgs(sinon.match.instanceOf(Uint8Array), defaultOptions).resolves(cid) const out = await cli(`object put ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/node.json`, { ipfs }) @@ -167,7 +167,7 @@ describe('object', () => { }) it('put from pipe', async () => { - const buf = Buffer.from('hello world') + const buf = uint8ArrayFromString('hello world') ipfs.object.put.withArgs(buf, defaultOptions).resolves(cid) @@ -197,7 +197,7 @@ describe('object', () => { }) it('should put an object with a timeout', async () => { - ipfs.object.put.withArgs(sinon.match.instanceOf(Buffer), { + ipfs.object.put.withArgs(sinon.match.instanceOf(Uint8Array), { ...defaultOptions, timeout: 1000 }).resolves(cid) diff --git a/packages/ipfs/test/cli/pin.js b/packages/ipfs/test/cli/pin.js index 3f8f9efe34..a67bba88b2 100644 --- a/packages/ipfs/test/cli/pin.js +++ b/packages/ipfs/test/cli/pin.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const CID = require('cids') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/ping.js b/packages/ipfs/test/cli/ping.js index 7c1481b9d9..3b9a057995 100644 --- a/packages/ipfs/test/cli/ping.js +++ b/packages/ipfs/test/cli/ping.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/progress-bar.js b/packages/ipfs/test/cli/progress-bar.js index 353222a2a2..b359dd8857 100644 --- a/packages/ipfs/test/cli/progress-bar.js +++ b/packages/ipfs/test/cli/progress-bar.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const createProgressBar = require('../../src/cli/utils').createProgressBar describe('progress bar', () => { diff --git a/packages/ipfs/test/cli/pubsub.js b/packages/ipfs/test/cli/pubsub.js index 94d4060995..026f593010 100644 --- a/packages/ipfs/test/cli/pubsub.js +++ b/packages/ipfs/test/cli/pubsub.js @@ -1,10 +1,10 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') describe('pubsub', () => { let ipfs @@ -95,7 +95,7 @@ describe('pubsub', () => { await cli(`pubsub pub ${subName} ${data}`, { ipfs }) - expect(ipfs.pubsub.publish.calledWith(subName, Buffer.from(data), defaultOptions)).to.be.true() + expect(ipfs.pubsub.publish.calledWith(subName, uint8ArrayFromString(data), defaultOptions)).to.be.true() }) it('should publish message with timeout', async () => { @@ -104,7 +104,7 @@ describe('pubsub', () => { await cli(`pubsub pub ${subName} ${data} --timeout=1s`, { ipfs }) - expect(ipfs.pubsub.publish.calledWith(subName, Buffer.from(data), { + expect(ipfs.pubsub.publish.calledWith(subName, uint8ArrayFromString(data), { ...defaultOptions, timeout: 1000 })).to.be.true() diff --git a/packages/ipfs/test/cli/refs-local.js b/packages/ipfs/test/cli/refs-local.js index 98da66970f..071308b90e 100644 --- a/packages/ipfs/test/cli/refs-local.js +++ b/packages/ipfs/test/cli/refs-local.js @@ -1,11 +1,11 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') const multibase = require('multibase') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const defaultOptions = { timeout: undefined @@ -52,7 +52,7 @@ describe('refs local', () => { const out = await cli('refs local --multihash', { ipfs }) const lines = out.split('\n') - expect(lines.includes(multibase.encoding('base32upper').encode(Buffer.from(ref)))).to.be.true() + expect(lines.includes(multibase.encoding('base32upper').encode(uint8ArrayFromString(ref)))).to.be.true() expect(lines.includes(err)).to.be.true() }) diff --git a/packages/ipfs/test/cli/refs.js b/packages/ipfs/test/cli/refs.js index cd9ed438ba..de6aaa1b9a 100644 --- a/packages/ipfs/test/cli/refs.js +++ b/packages/ipfs/test/cli/refs.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const CID = require('cids') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/cli/repo.js b/packages/ipfs/test/cli/repo.js index 7d51a32916..7bc984bf13 100644 --- a/packages/ipfs/test/cli/repo.js +++ b/packages/ipfs/test/cli/repo.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') const BigNumber = require('bignumber.js') diff --git a/packages/ipfs/test/cli/swarm.js b/packages/ipfs/test/cli/swarm.js index bbcd238344..a57cc3485a 100644 --- a/packages/ipfs/test/cli/swarm.js +++ b/packages/ipfs/test/cli/swarm.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') const ma = require('multiaddr') diff --git a/packages/ipfs/test/cli/version.js b/packages/ipfs/test/cli/version.js index bb01b0ab9e..f4885036dd 100644 --- a/packages/ipfs/test/cli/version.js +++ b/packages/ipfs/test/cli/version.js @@ -3,7 +3,7 @@ 'use strict' const os = require('os') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const cli = require('../utils/cli') const sinon = require('sinon') diff --git a/packages/ipfs/test/core/add-all.spec.js b/packages/ipfs/test/core/add-all.spec.js index c2bb48930a..e0de81b46c 100644 --- a/packages/ipfs/test/core/add-all.spec.js +++ b/packages/ipfs/test/core/add-all.spec.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const utils = require('../../src/core/components/add-all/utils') describe('add-all/utils', () => { diff --git a/packages/ipfs/test/core/bitswap.spec.js b/packages/ipfs/test/core/bitswap.spec.js index da1b4fa123..747fe409c0 100644 --- a/packages/ipfs/test/core/bitswap.spec.js +++ b/packages/ipfs/test/core/bitswap.spec.js @@ -3,16 +3,16 @@ const { nanoid } = require('nanoid') const pmap = require('p-map') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const Block = require('ipld-block') const multihashing = require('multihashing-async') const CID = require('cids') const concat = require('it-concat') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const factory = require('../utils/factory') const makeBlock = async () => { - const d = Buffer.from(`IPFS is awesome ${nanoid()}`) + const d = uint8ArrayFromString(`IPFS is awesome ${nanoid()}`) const h = await multihashing(d, 'sha2-256') return new Block(d, new CID(h)) @@ -65,7 +65,7 @@ describe('bitswap', function () { it('2 peers', async () => { // TODO make this test more interesting (10Mb file) // TODO remove randomness from the test - const content = Buffer.from(`I love IPFS <3 ${nanoid()}`) + const content = uint8ArrayFromString(`I love IPFS <3 ${nanoid()}`) const remote = (await df.spawn({ type: 'js' })).api const proc = (await df.spawn({ type: 'proc' })).api proc.swarm.connect(remote.peerId.addresses[0]) diff --git a/packages/ipfs/test/core/block.spec.js b/packages/ipfs/test/core/block.spec.js index 2741b3ec51..99bdf6fc27 100644 --- a/packages/ipfs/test/core/block.spec.js +++ b/packages/ipfs/test/core/block.spec.js @@ -2,10 +2,10 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const { nanoid } = require('nanoid') const all = require('it-all') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const factory = require('../utils/factory') describe('block', () => { @@ -28,7 +28,7 @@ describe('block', () => { describe('put', () => { it('should not error when passed null options', () => { - return ipfs.block.put(Buffer.from(nanoid()), null) + return ipfs.block.put(uint8ArrayFromString(nanoid()), null) }) }) @@ -48,7 +48,7 @@ describe('block', () => { }) it('should not error when passed null options', async () => { - const block = await ipfs.block.put(Buffer.from(nanoid())) + const block = await ipfs.block.put(uint8ArrayFromString(nanoid())) return ipfs.block.stat(block.cid, null) }) }) diff --git a/packages/ipfs/test/core/bootstrap.spec.js b/packages/ipfs/test/core/bootstrap.spec.js index 252d78667a..f1fadffbcd 100644 --- a/packages/ipfs/test/core/bootstrap.spec.js +++ b/packages/ipfs/test/core/bootstrap.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const factory = require('../utils/factory') const { isBrowser, isWebWorker } = require('ipfs-utils/src/env') diff --git a/packages/ipfs/test/core/circuit-relay.spec.js b/packages/ipfs/test/core/circuit-relay.spec.js index 2fed2390f2..b7ba3700a0 100644 --- a/packages/ipfs/test/core/circuit-relay.spec.js +++ b/packages/ipfs/test/core/circuit-relay.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const concat = require('it-concat') const randomBytes = require('iso-random-stream/src/random') const factory = require('../utils/factory') diff --git a/packages/ipfs/test/core/create-node.spec.js b/packages/ipfs/test/core/create-node.spec.js index 4e66babfd4..8e982fddc5 100644 --- a/packages/ipfs/test/core/create-node.spec.js +++ b/packages/ipfs/test/core/create-node.spec.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const sinon = require('sinon') const { isNode } = require('ipfs-utils/src/env') const tmpDir = require('ipfs-utils/src/temp-dir') diff --git a/packages/ipfs/test/core/dag.spec.js b/packages/ipfs/test/core/dag.spec.js index f2d22316f5..3ef155c36e 100644 --- a/packages/ipfs/test/core/dag.spec.js +++ b/packages/ipfs/test/core/dag.spec.js @@ -2,10 +2,10 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const all = require('it-all') const factory = require('../utils/factory') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') describe('dag', function () { this.timeout(10 * 1000) @@ -26,7 +26,7 @@ describe('dag', function () { }) it('should throw error for invalid buffer CID input', () => { - return expect(ipfs.dag.get(Buffer.from('INVALID CID'))) + return expect(ipfs.dag.get(uint8ArrayFromString('INVALID CID'))) .to.eventually.be.rejected() .and.to.have.property('code').that.equals('ERR_INVALID_CID') }) diff --git a/packages/ipfs/test/core/dht.spec.js b/packages/ipfs/test/core/dht.spec.js index 88899b6c08..b2509cb960 100644 --- a/packages/ipfs/test/core/dht.spec.js +++ b/packages/ipfs/test/core/dht.spec.js @@ -2,10 +2,10 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const { isNode } = require('ipfs-utils/src/env') const factory = require('../utils/factory') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') describe('dht', () => { describe('enabled by config', () => { @@ -31,7 +31,7 @@ describe('dht', () => { describe('put', () => { it('should put a value when enabled', async () => { - await expect(ipfs.dht.put(Buffer.from('a'), Buffer.from('b'))) + await expect(ipfs.dht.put(uint8ArrayFromString('a'), uint8ArrayFromString('b'))) .to.eventually.be.undefined() }) }) @@ -71,7 +71,7 @@ describe('dht', () => { describe('put', () => { it('should error when DHT not available', async () => { - await expect(ipfs.dht.put(Buffer.from('a'), Buffer.from('b'))) + await expect(ipfs.dht.put(uint8ArrayFromString('a'), uint8ArrayFromString('b'))) .to.eventually.be.rejected() .and.to.have.property('code', 'ERR_NOT_ENABLED') }) @@ -94,7 +94,7 @@ describe('dht', () => { describe('put', () => { it('should error when DHT not available', async () => { - await expect(ipfs.dht.put(Buffer.from('a'), Buffer.from('b'))) + await expect(ipfs.dht.put(uint8ArrayFromString('a'), uint8ArrayFromString('b'))) .to.eventually.be.rejected() .and.to.have.property('code', 'ERR_NOT_ENABLED') }) diff --git a/packages/ipfs/test/core/exports.spec.js b/packages/ipfs/test/core/exports.spec.js index 3022d975cd..7bf415337a 100644 --- a/packages/ipfs/test/core/exports.spec.js +++ b/packages/ipfs/test/core/exports.spec.js @@ -10,8 +10,7 @@ const multihashing = require('multihashing-async') const multihash = multihashing.multihash const multicodec = require('multicodec') const PeerId = require('peer-id') -const { Buffer } = require('buffer') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const Ipfs = require('../../') @@ -19,7 +18,6 @@ describe('exports', () => { it('should export the expected types and utilities', () => { expect(Ipfs.crypto).to.equal(crypto) expect(Ipfs.isIPFS).to.equal(isIPFS) - expect(Ipfs.Buffer).to.equal(Buffer) expect(Ipfs.CID).to.equal(CID) expect(Ipfs.multiaddr).to.equal(multiaddr) expect(Ipfs.multibase).to.equal(multibase) diff --git a/packages/ipfs/test/core/files-sharding.spec.js b/packages/ipfs/test/core/files-sharding.spec.js index 4d3270d656..f4c492bf4b 100644 --- a/packages/ipfs/test/core/files-sharding.spec.js +++ b/packages/ipfs/test/core/files-sharding.spec.js @@ -2,9 +2,9 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const last = require('it-last') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const factory = require('../utils/factory') describe('files directory (sharding tests)', function () { @@ -13,7 +13,7 @@ describe('files directory (sharding tests)', function () { const testFiles = Array.from(Array(1005), (_, i) => ({ path: 'test-folder/' + i, - content: Buffer.from('some content ' + i) + content: uint8ArrayFromString('some content ' + i) })) describe('without sharding', () => { diff --git a/packages/ipfs/test/core/files.spec.js b/packages/ipfs/test/core/files.spec.js index 2b50ad1e28..395c512cec 100644 --- a/packages/ipfs/test/core/files.spec.js +++ b/packages/ipfs/test/core/files.spec.js @@ -2,9 +2,9 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const { nanoid } = require('nanoid') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const all = require('it-all') const factory = require('../utils/factory') @@ -31,11 +31,11 @@ describe('files', function () { describe('add', () => { it('should not error when passed null options', async () => { - await ipfs.add(Buffer.from(nanoid()), null) + await ipfs.add(uint8ArrayFromString(nanoid()), null) }) it('should add a file with a v1 CID', async () => { - const file = await ipfs.add(Buffer.from([0, 1, 2]), { + const file = await ipfs.add(Uint8Array.from([0, 1, 2]), { cidVersion: 1 }) diff --git a/packages/ipfs/test/core/gc.spec.js b/packages/ipfs/test/core/gc.spec.js index fce8319778..d826afecea 100644 --- a/packages/ipfs/test/core/gc.spec.js +++ b/packages/ipfs/test/core/gc.spec.js @@ -2,8 +2,8 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const { Buffer } = require('buffer') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') const factory = require('../utils/factory') const pEvent = require('p-event') @@ -41,16 +41,16 @@ describe.skip('gc', function () { const df = factory() const fixtures = [{ path: 'test/my/path1', - content: Buffer.from('path1') + content: uint8ArrayFromString('path1') }, { path: 'test/my/path2', - content: Buffer.from('path2') + content: uint8ArrayFromString('path2') }, { path: 'test/my/path3', - content: Buffer.from('path3') + content: uint8ArrayFromString('path3') }, { path: 'test/my/path4', - content: Buffer.from('path4') + content: uint8ArrayFromString('path4') }] let ipfsd @@ -81,8 +81,8 @@ describe.skip('gc', function () { resToCid: (res) => res.toString() }, { name: 'block put', - add1: () => ipfs.block.put(Buffer.from('block put 1'), null), - add2: () => ipfs.block.put(Buffer.from('block put 2'), null), + add1: () => ipfs.block.put(uint8ArrayFromString('block put 1'), null), + add2: () => ipfs.block.put(uint8ArrayFromString('block put 2'), null), resToCid: (res) => res.cid.toString() }] @@ -150,8 +150,8 @@ describe.skip('gc', function () { it('garbage collection should wait for pending block rm to finish', async () => { // Add two blocks so that we can remove them - const cid1 = (await ipfs.block.put(Buffer.from('block to rm 1'), null)).cid - const cid2 = (await ipfs.block.put(Buffer.from('block to rm 2'), null)).cid + const cid1 = (await ipfs.block.put(uint8ArrayFromString('block to rm 1'), null)).cid + const cid2 = (await ipfs.block.put(uint8ArrayFromString('block to rm 2'), null)).cid // Remove first block from IPFS // Note: block rm will take a write lock @@ -193,8 +193,8 @@ describe.skip('gc', function () { it('garbage collection should wait for pending pin add to finish', async () => { // Add two blocks so that we can pin them - const cid1 = (await ipfs.block.put(Buffer.from('block to pin add 1'), null)).cid - const cid2 = (await ipfs.block.put(Buffer.from('block to pin add 2'), null)).cid + const cid1 = (await ipfs.block.put(uint8ArrayFromString('block to pin add 1'), null)).cid + const cid2 = (await ipfs.block.put(uint8ArrayFromString('block to pin add 2'), null)).cid // Pin first block // Note: pin add will take a read lock @@ -226,8 +226,8 @@ describe.skip('gc', function () { it('garbage collection should wait for pending pin rm to finish', async () => { // Add two blocks so that we can pin them - const cid1 = (await ipfs.block.put(Buffer.from('block to pin rm 1'), null)).cid - const cid2 = (await ipfs.block.put(Buffer.from('block to pin rm 2'), null)).cid + const cid1 = (await ipfs.block.put(uint8ArrayFromString('block to pin rm 1'), null)).cid + const cid2 = (await ipfs.block.put(uint8ArrayFromString('block to pin rm 2'), null)).cid // Pin blocks await ipfs.pin.add(cid1) diff --git a/packages/ipfs/test/core/id.spec.js b/packages/ipfs/test/core/id.spec.js index 5793e679e1..34c072e21b 100644 --- a/packages/ipfs/test/core/id.spec.js +++ b/packages/ipfs/test/core/id.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const multiaddr = require('multiaddr') const { isBrowser, isWebWorker } = require('ipfs-utils/src/env') const factory = require('../utils/factory') diff --git a/packages/ipfs/test/core/init.spec.js b/packages/ipfs/test/core/init.spec.js index 782b7dd977..b7a92b83cb 100644 --- a/packages/ipfs/test/core/init.spec.js +++ b/packages/ipfs/test/core/init.spec.js @@ -2,9 +2,9 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const { isNode } = require('ipfs-utils/src/env') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const { nanoid } = require('nanoid') const PeerId = require('peer-id') const { supportedKeys } = require('libp2p-crypto/src/keys') @@ -120,7 +120,7 @@ describe('init', function () { await ipfs.init({ bits: 512, emptyRepo: true }) // Should not have default assets - const multihash = Buffer.from('12205e7c3ce237f936c76faf625e90f7751a9f5eeb048f59873303c215e9cce87599', 'hex') + const multihash = uint8ArrayFromString('12205e7c3ce237f936c76faf625e90f7751a9f5eeb048f59873303c215e9cce87599', 'hex') await expect(ipfs.object.get(multihash, {})).to.eventually.be.rejected() }) diff --git a/packages/ipfs/test/core/ipld.spec.js b/packages/ipfs/test/core/ipld.spec.js index 2896c3232b..dcc398398c 100644 --- a/packages/ipfs/test/core/ipld.spec.js +++ b/packages/ipfs/test/core/ipld.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const factory = require('../utils/factory') const ipldDagPb = require('ipld-dag-pb') @@ -31,7 +31,7 @@ describe('ipld', function () { hashAlg: 'sha2-256' }) - const dagPbNode = new ipldDagPb.DAGNode(Buffer.alloc(0), [], 0) + const dagPbNode = new ipldDagPb.DAGNode(new Uint8Array(0), [], 0) const cid2 = await ipfs.dag.put(dagPbNode, { format: 'dag-pb', hashAlg: 'sha2-256' diff --git a/packages/ipfs/test/core/kad-dht.node.js b/packages/ipfs/test/core/kad-dht.node.js index e04403a1ac..bf830731ba 100644 --- a/packages/ipfs/test/core/kad-dht.node.js +++ b/packages/ipfs/test/core/kad-dht.node.js @@ -2,9 +2,9 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const path = require('path') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const concat = require('it-concat') const factory = require('../utils/factory') @@ -56,7 +56,7 @@ describe.skip('kad-dht is routing content and peers correctly', () => { this.timeout(30 * 1000) const file = { path: 'testfile1.txt', - content: Buffer.from('hello kad 1') + content: uint8ArrayFromString('hello kad 1') } const fileAdded = await nodeB.add(file) @@ -69,7 +69,7 @@ describe.skip('kad-dht is routing content and peers correctly', () => { this.timeout(30 * 1000) const file = { path: 'testfile2.txt', - content: Buffer.from('hello kad 2') + content: uint8ArrayFromString('hello kad 2') } const fileAdded = await nodeC.add(file) diff --git a/packages/ipfs/test/core/key-exchange.spec.js b/packages/ipfs/test/core/key-exchange.spec.js index b8d7642b63..876b362195 100644 --- a/packages/ipfs/test/core/key-exchange.spec.js +++ b/packages/ipfs/test/core/key-exchange.spec.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const { nanoid } = require('nanoid') const factory = require('../utils/factory') diff --git a/packages/ipfs/test/core/libp2p.spec.js b/packages/ipfs/test/core/libp2p.spec.js index 27c8928aa4..f192b98c3c 100644 --- a/packages/ipfs/test/core/libp2p.spec.js +++ b/packages/ipfs/test/core/libp2p.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const MemoryStore = require('interface-datastore').MemoryDatastore const PeerId = require('peer-id') const Libp2p = require('libp2p') diff --git a/packages/ipfs/test/core/mfs-preload.spec.js b/packages/ipfs/test/core/mfs-preload.spec.js index a2c5e5316f..2db9989997 100644 --- a/packages/ipfs/test/core/mfs-preload.spec.js +++ b/packages/ipfs/test/core/mfs-preload.spec.js @@ -1,17 +1,17 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const delay = require('delay') const multihashing = require('multihashing-async') const { nanoid } = require('nanoid') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const CID = require('cids') const waitFor = require('../utils/wait-for') const mfsPreload = require('../../src/core/mfs-preload') const fakeCid = async () => { - const mh = await multihashing(Buffer.from(nanoid()), 'sha2-256') + const mh = await multihashing(uint8ArrayFromString(nanoid()), 'sha2-256') return new CID(mh) } diff --git a/packages/ipfs/test/core/name-pubsub.js b/packages/ipfs/test/core/name-pubsub.js index 93cb421184..20728d80be 100644 --- a/packages/ipfs/test/core/name-pubsub.js +++ b/packages/ipfs/test/core/name-pubsub.js @@ -3,10 +3,7 @@ 'use strict' const { nanoid } = require('nanoid') -const { Buffer } = require('buffer') -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const base64url = require('base64url') -const { fromB58String } = require('multihashing-async').multihash +const { expect } = require('aegir/utils/chai') const PeerId = require('peer-id') const { isNode } = require('ipfs-utils/src/env') const ipns = require('ipns') @@ -14,6 +11,8 @@ const delay = require('delay') const last = require('it-last') const waitFor = require('../utils/wait-for') const factory = require('../utils/factory') +const uint8ArrayToString = require('uint8arrays/to-string') +const uint8ArrayFromString = require('uint8arrays/from-string') const namespace = '/record/' const ipfsRef = '/ipfs/QmPFVLPmp9zv5Z5KUqLhe2EivAGccQW2r7M7jhVJGLZoZU' @@ -66,8 +65,8 @@ describe('name-pubsub', function () { return subscribed === true } - const keys = ipns.getIdKeys(fromB58String(idA.id)) - const topic = `${namespace}${base64url.encode(keys.routingKey.toBuffer())}` + const keys = ipns.getIdKeys(uint8ArrayFromString(idA.id, 'base58btc')) + const topic = `${namespace}${uint8ArrayToString(keys.routingKey.uint8Array(), 'base64url')}` await expect(last(nodeB.name.resolve(idA.id))) .to.eventually.be.rejected() @@ -90,7 +89,7 @@ describe('name-pubsub', function () { it('should self resolve, publish and then resolve correctly', async function () { this.timeout(6000) const emptyDirCid = '/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' - const { path } = await nodeA.add(Buffer.from('pubsub records')) + const { path } = await nodeA.add(uint8ArrayFromString('pubsub records')) const resolvesEmpty = await last(nodeB.name.resolve(idB.id)) expect(resolvesEmpty).to.be.eq(emptyDirCid) @@ -124,7 +123,7 @@ describe('name-pubsub', function () { function checkMessage (msg) { publishedMessage = msg publishedMessageData = ipns.unmarshal(msg.data) - publishedMessageDataValue = publishedMessageData.value.toString('utf8') + publishedMessageDataValue = uint8ArrayToString(publishedMessageData.value) } const alreadySubscribed = () => { @@ -137,8 +136,8 @@ describe('name-pubsub', function () { size: 2048 }) - const keys = ipns.getIdKeys(fromB58String(testAccount.id)) - const topic = `${namespace}${base64url.encode(keys.routingKey.toBuffer())}` + const keys = ipns.getIdKeys(uint8ArrayFromString(testAccount.id, 'base58btc')) + const topic = `${namespace}${uint8ArrayToString(keys.routingKey.uint8Array(), 'base64url')}` await nodeB.pubsub.subscribe(topic, checkMessage) await nodeA.name.publish(ipfsRef, { resolve: false, key: testAccountName }) diff --git a/packages/ipfs/test/core/name.spec.js b/packages/ipfs/test/core/name.spec.js index 993fd868ba..35e901252f 100644 --- a/packages/ipfs/test/core/name.spec.js +++ b/packages/ipfs/test/core/name.spec.js @@ -2,7 +2,7 @@ 'use strict' const { nanoid } = require('nanoid') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const sinon = require('sinon') const delay = require('delay') const { Key } = require('interface-datastore') diff --git a/packages/ipfs/test/core/object.spec.js b/packages/ipfs/test/core/object.spec.js index ec7e93d639..54ede829e5 100644 --- a/packages/ipfs/test/core/object.spec.js +++ b/packages/ipfs/test/core/object.spec.js @@ -2,9 +2,9 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const { nanoid } = require('nanoid') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const factory = require('../utils/factory') describe('object', function () { @@ -27,21 +27,21 @@ describe('object', function () { }) it('should not error when passed null options', async () => { - const cid = await ipfs.object.put(Buffer.from(nanoid())) + const cid = await ipfs.object.put(uint8ArrayFromString(nanoid())) await ipfs.object.get(cid) }) }) describe('put', () => { it('should not error when passed null options', () => { - return ipfs.object.put(Buffer.from(nanoid()), null) + return ipfs.object.put(uint8ArrayFromString(nanoid()), null) }) }) describe('patch.addLink', () => { it('should not error when passed null options', async () => { - const aCid = await ipfs.object.put(Buffer.from(nanoid())) - const bCid = await ipfs.object.put(Buffer.from(nanoid())) + const aCid = await ipfs.object.put(uint8ArrayFromString(nanoid())) + const bCid = await ipfs.object.put(uint8ArrayFromString(nanoid())) const bNode = await ipfs.object.get(bCid) const link = { @@ -56,8 +56,8 @@ describe('object', function () { describe('patch.rmLink', () => { it('should not error when passed null options', async () => { - const aCid = await ipfs.object.put(Buffer.from(nanoid())) - const bCid = await ipfs.object.put(Buffer.from(nanoid())) + const aCid = await ipfs.object.put(uint8ArrayFromString(nanoid())) + const bCid = await ipfs.object.put(uint8ArrayFromString(nanoid())) const bNode = await ipfs.object.get(bCid) const cCid = await ipfs.object.patch.addLink(aCid, { @@ -73,15 +73,15 @@ describe('object', function () { describe('patch.appendData', () => { it('should not error when passed null options', async () => { - const cid = await ipfs.object.put(Buffer.from(nanoid()), null) - await ipfs.object.patch.appendData(cid, Buffer.from(nanoid()), null) + const cid = await ipfs.object.put(uint8ArrayFromString(nanoid()), null) + await ipfs.object.patch.appendData(cid, uint8ArrayFromString(nanoid()), null) }) }) describe('patch.setData', () => { it('should not error when passed null options', async () => { - const cid = await ipfs.object.put(Buffer.from(nanoid()), null) - await ipfs.object.patch.setData(cid, Buffer.from(nanoid()), null) + const cid = await ipfs.object.put(uint8ArrayFromString(nanoid()), null) + await ipfs.object.patch.setData(cid, uint8ArrayFromString(nanoid()), null) }) }) }) diff --git a/packages/ipfs/test/core/ping.spec.js b/packages/ipfs/test/core/ping.spec.js index 901eb620d3..f1740414fc 100644 --- a/packages/ipfs/test/core/ping.spec.js +++ b/packages/ipfs/test/core/ping.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const all = require('it-all') const factory = require('../utils/factory') diff --git a/packages/ipfs/test/core/preload.spec.js b/packages/ipfs/test/core/preload.spec.js index 65c6dfcecd..954c26ca41 100644 --- a/packages/ipfs/test/core/preload.spec.js +++ b/packages/ipfs/test/core/preload.spec.js @@ -2,8 +2,8 @@ 'use strict' const { nanoid } = require('nanoid') -const { Buffer } = require('buffer') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const uint8ArrayFromString = require('uint8arrays/from-string') +const { expect } = require('aegir/utils/chai') const all = require('it-all') const MockPreloadNode = require('../utils/mock-preload-node-utils') const IPFS = require('../../src') @@ -37,7 +37,7 @@ describe('preload', () => { it('should preload content added with add', async function () { this.timeout(50 * 1000) - const res = await ipfs.add(Buffer.from(nanoid())) + const res = await ipfs.add(uint8ArrayFromString(nanoid())) await MockPreloadNode.waitForCids(res.cid) }) @@ -45,11 +45,11 @@ describe('preload', () => { this.timeout(50 * 1000) const res = await all(ipfs.addAll([{ - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }, { - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }, { - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }])) await MockPreloadNode.waitForCids(res.map(file => file.cid)) @@ -60,13 +60,13 @@ describe('preload', () => { const res = await all(ipfs.addAll([{ path: 'dir0/dir1/file0', - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }, { path: 'dir0/dir1/file1', - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }, { path: 'dir0/file2', - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }])) const rootDir = res.find(file => file.path === 'dir0') @@ -80,13 +80,13 @@ describe('preload', () => { const res = await all(ipfs.addAll([{ path: 'dir0/dir1/file0', - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }, { path: 'dir0/dir1/file1', - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }, { path: 'dir0/file2', - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }], { wrapWithDirectory: true })) const wrappingDir = res.find(file => file.path === '') @@ -97,14 +97,14 @@ describe('preload', () => { it('should preload content retrieved with cat', async function () { this.timeout(50 * 1000) - const res = await ipfs.add(Buffer.from(nanoid()), { preload: false }) + const res = await ipfs.add(uint8ArrayFromString(nanoid()), { preload: false }) await all(ipfs.cat(res.cid)) await MockPreloadNode.waitForCids(res.cid) }) it('should preload content retrieved with get', async function () { this.timeout(50 * 1000) - const res = await ipfs.add(Buffer.from(nanoid()), { preload: false }) + const res = await ipfs.add(uint8ArrayFromString(nanoid()), { preload: false }) await all(ipfs.get(res.cid)) await MockPreloadNode.waitForCids(res.cid) }) @@ -114,13 +114,13 @@ describe('preload', () => { const res = await all(ipfs.addAll([{ path: 'dir0/dir1/file0', - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }, { path: 'dir0/dir1/file1', - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }, { path: 'dir0/file2', - content: Buffer.from(nanoid()) + content: uint8ArrayFromString(nanoid()) }], { wrapWithDirectory: true })) const wrappingDir = res.find(file => file.path === '') @@ -141,7 +141,7 @@ describe('preload', () => { it('should preload content added with object.put', async function () { this.timeout(50 * 1000) - const cid = await ipfs.object.put({ Data: Buffer.from(nanoid()), Links: [] }) + const cid = await ipfs.object.put({ Data: uint8ArrayFromString(nanoid()), Links: [] }) await MockPreloadNode.waitForCids(cid) }) @@ -149,7 +149,7 @@ describe('preload', () => { this.timeout(50 * 1000) const createNode = async () => { - const cid = await ipfs.object.put({ Data: Buffer.from(nanoid()), Links: [] }) + const cid = await ipfs.object.put({ Data: uint8ArrayFromString(nanoid()), Links: [] }) const node = await ipfs.object.get(cid) return { cid, node } } @@ -168,11 +168,11 @@ describe('preload', () => { it('should preload content added with object.patch.rmLink', async function () { this.timeout(50 * 1000) - const linkCid = await ipfs.object.put({ Data: Buffer.from(nanoid()), Links: [] }) + const linkCid = await ipfs.object.put({ Data: uint8ArrayFromString(nanoid()), Links: [] }) const linkNode = await ipfs.object.get(linkCid) const parentCid = await ipfs.object.put({ - Data: Buffer.from(nanoid()), + Data: uint8ArrayFromString(nanoid()), Links: [{ name: 'link', cid: linkCid, @@ -186,15 +186,15 @@ describe('preload', () => { it('should preload content added with object.patch.setData', async function () { this.timeout(50 * 1000) - const originalCid = await ipfs.object.put({ Data: Buffer.from(nanoid()), Links: [] }) - const patchedCid = await ipfs.object.patch.setData(originalCid, Buffer.from(nanoid())) + const originalCid = await ipfs.object.put({ Data: uint8ArrayFromString(nanoid()), Links: [] }) + const patchedCid = await ipfs.object.patch.setData(originalCid, uint8ArrayFromString(nanoid())) await MockPreloadNode.waitForCids(patchedCid) }) it('should preload content added with object.patch.appendData', async function () { this.timeout(50 * 1000) - const originalCid = await ipfs.object.put({ Data: Buffer.from(nanoid()), Links: [] }) - const patchedCid = await ipfs.object.patch.appendData(originalCid, Buffer.from(nanoid())) + const originalCid = await ipfs.object.put({ Data: uint8ArrayFromString(nanoid()), Links: [] }) + const patchedCid = await ipfs.object.patch.appendData(originalCid, uint8ArrayFromString(nanoid())) await MockPreloadNode.waitForCids(patchedCid) }) @@ -207,20 +207,20 @@ describe('preload', () => { it('should preload content added with block.put', async function () { this.timeout(50 * 1000) - const block = await ipfs.block.put(Buffer.from(nanoid())) + const block = await ipfs.block.put(uint8ArrayFromString(nanoid())) await MockPreloadNode.waitForCids(block.cid) }) it('should preload content retrieved with block.get', async function () { this.timeout(50 * 1000) - const block = await ipfs.block.put(Buffer.from(nanoid()), { preload: false }) + const block = await ipfs.block.put(uint8ArrayFromString(nanoid()), { preload: false }) await ipfs.block.get(block.cid) await MockPreloadNode.waitForCids(block.cid) }) it('should preload content retrieved with block.stat', async function () { this.timeout(50 * 1000) - const block = await ipfs.block.put(Buffer.from(nanoid()), { preload: false }) + const block = await ipfs.block.put(uint8ArrayFromString(nanoid()), { preload: false }) await ipfs.block.stat(block.cid) await MockPreloadNode.waitForCids(block.cid) }) @@ -242,7 +242,7 @@ describe('preload', () => { }) it('should preload content retrieved with files.ls', async () => { - const res = await ipfs.add({ path: `/t/${nanoid()}`, content: Buffer.from(nanoid()) }) + const res = await ipfs.add({ path: `/t/${nanoid()}`, content: uint8ArrayFromString(nanoid()) }) const dirCid = res.cid await MockPreloadNode.waitForCids(dirCid) await MockPreloadNode.clearPreloadCids() @@ -251,7 +251,7 @@ describe('preload', () => { }) it('should preload content retrieved with files.ls by CID', async () => { - const res = await ipfs.add({ path: `/t/${nanoid()}`, content: Buffer.from(nanoid()) }) + const res = await ipfs.add({ path: `/t/${nanoid()}`, content: uint8ArrayFromString(nanoid()) }) const dirCid = res.cid await MockPreloadNode.waitForCids(dirCid) await MockPreloadNode.clearPreloadCids() @@ -260,7 +260,7 @@ describe('preload', () => { }) it('should preload content retrieved with files.read', async () => { - const { cid } = await ipfs.add(Buffer.from(nanoid())) + const { cid } = await ipfs.add(uint8ArrayFromString(nanoid())) await MockPreloadNode.waitForCids(cid) await MockPreloadNode.clearPreloadCids() await ipfs.files.read(`/ipfs/${cid}`) @@ -268,7 +268,7 @@ describe('preload', () => { }) it('should preload content retrieved with files.stat', async () => { - const { cid: fileCid } = await ipfs.add(Buffer.from(nanoid())) + const { cid: fileCid } = await ipfs.add(uint8ArrayFromString(nanoid())) await MockPreloadNode.waitForCids(fileCid) await MockPreloadNode.clearPreloadCids() await ipfs.files.stat(`/ipfs/${fileCid}`) @@ -304,7 +304,7 @@ describe('preload disabled', function () { after(() => repo.teardown()) it('should not preload if disabled', async () => { - const { cid } = await ipfs.add(Buffer.from(nanoid())) + const { cid } = await ipfs.add(uint8ArrayFromString(nanoid())) return expect(MockPreloadNode.waitForCids(cid)) .to.eventually.be.rejected() diff --git a/packages/ipfs/test/core/pubsub.spec.js b/packages/ipfs/test/core/pubsub.spec.js index c9a967fc11..478c6505dd 100644 --- a/packages/ipfs/test/core/pubsub.spec.js +++ b/packages/ipfs/test/core/pubsub.spec.js @@ -3,8 +3,8 @@ 'use strict' const { nanoid } = require('nanoid') -const { Buffer } = require('buffer') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const uint8ArrayFromString = require('uint8arrays/from-string') +const { expect } = require('aegir/utils/chai') const IPFS = require('../../src') const createTempRepo = require('../utils/create-repo-nodejs') @@ -57,7 +57,7 @@ describe('pubsub disabled', () => { it('should not allow publish if disabled', async () => { const topic = nanoid() - const msg = Buffer.from(nanoid()) + const msg = uint8ArrayFromString(nanoid()) await expect(ipfs.pubsub.publish(topic, msg)) .to.eventually.be.rejected() diff --git a/packages/ipfs/test/core/stats.spec.js b/packages/ipfs/test/core/stats.spec.js index 6b8bc4bd22..ee7e208bb1 100644 --- a/packages/ipfs/test/core/stats.spec.js +++ b/packages/ipfs/test/core/stats.spec.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const all = require('it-all') const factory = require('../utils/factory') diff --git a/packages/ipfs/test/core/utils.js b/packages/ipfs/test/core/utils.js index d3a5313bf5..7c657b02af 100644 --- a/packages/ipfs/test/core/utils.js +++ b/packages/ipfs/test/core/utils.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const fs = require('fs') const { fromB58String } = require('multihashing-async').multihash diff --git a/packages/ipfs/test/gateway/index.js b/packages/ipfs/test/gateway/index.js index f4bf98061d..43b7771cfe 100644 --- a/packages/ipfs/test/gateway/index.js +++ b/packages/ipfs/test/gateway/index.js @@ -2,8 +2,8 @@ /* eslint dot-notation: 0, dot-notation: 0, quote-props: 0 */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const { Buffer } = require('buffer') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') const Daemon = require('../../src/cli/daemon') const loadFixture = require('aegir/fixtures') const os = require('os') @@ -77,7 +77,7 @@ describe('HTTP Gateway', function () { // Qme79tX2bViL26vNjPsF3DP1R9rMKMvnPYJiKTTKPrXJjq await http.api._ipfs.add(bigFile) // QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o - await http.api._ipfs.add(Buffer.from('hello world' + '\n'), { cidVersion: 0 }) + await http.api._ipfs.add(uint8ArrayFromString('hello world' + '\n'), { cidVersion: 0 }) // QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ await http.api._ipfs.add([content('cat-folder/cat.jpg')]) // QmVZoGxDvKM9KExc8gaL4uTbhdNtWhzQR7ndrY7J1gWs3F @@ -138,7 +138,7 @@ describe('HTTP Gateway', function () { }) expect(res).to.have.property('statusCode', 200) - expect(res.rawPayload).to.eql(Buffer.from('hello world' + '\n')) + expect(res.rawPayload).to.eql(uint8ArrayFromString('hello world' + '\n')) expect(res.payload).to.equal('hello world' + '\n') expect(res.headers['cache-control']).to.equal('public, max-age=29030400, immutable') expect(res.headers['content-length']).to.equal(res.rawPayload.length).to.equal(12) @@ -171,7 +171,7 @@ describe('HTTP Gateway', function () { url: '/ipfs/TO-DO' }, (res) => { expect(res).to.have.property('statusCode', 200) - expect(res.rawPayload).to.eql(Buffer.from('hello world' + '\n')) + expect(res.rawPayload).to.eql(uint8ArrayFromString('hello world' + '\n')) expect(res.payload).to.equal('hello world' + '\n') expect(res.headers.etag).to.equal(TO-DO) expect(res.headers['x-ipfs-path']).to.equal(TO-DO) diff --git a/packages/ipfs/test/http-api/inject/bitswap.js b/packages/ipfs/test/http-api/inject/bitswap.js index e4e29d15ad..79f62b1c34 100644 --- a/packages/ipfs/test/http-api/inject/bitswap.js +++ b/packages/ipfs/test/http-api/inject/bitswap.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const CID = require('cids') const sinon = require('sinon') const testHttpMethod = require('../../utils/test-http-method') diff --git a/packages/ipfs/test/http-api/inject/block.js b/packages/ipfs/test/http-api/inject/block.js index 592a05f9ad..3379c43e60 100644 --- a/packages/ipfs/test/http-api/inject/block.js +++ b/packages/ipfs/test/http-api/inject/block.js @@ -2,9 +2,8 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const FormData = require('form-data') -const { Buffer } = require('buffer') const streamToPromise = require('stream-to-promise') const multibase = require('multibase') const testHttpMethod = require('../../utils/test-http-method') diff --git a/packages/ipfs/test/http-api/inject/bootstrap.js b/packages/ipfs/test/http-api/inject/bootstrap.js index 999c8602d0..8aa1de2f6c 100644 --- a/packages/ipfs/test/http-api/inject/bootstrap.js +++ b/packages/ipfs/test/http-api/inject/bootstrap.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const qs = require('qs') const defaultList = require('../../../src/core/runtime/config-nodejs.js')().Bootstrap const testHttpMethod = require('../../utils/test-http-method') diff --git a/packages/ipfs/test/http-api/inject/browser-headers.js b/packages/ipfs/test/http-api/inject/browser-headers.js index 3ccb99694e..6e69a62224 100644 --- a/packages/ipfs/test/http-api/inject/browser-headers.js +++ b/packages/ipfs/test/http-api/inject/browser-headers.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const sinon = require('sinon') const http = require('../../utils/http') diff --git a/packages/ipfs/test/http-api/inject/config.js b/packages/ipfs/test/http-api/inject/config.js index 62b0d5be6e..255bec9086 100644 --- a/packages/ipfs/test/http-api/inject/config.js +++ b/packages/ipfs/test/http-api/inject/config.js @@ -1,9 +1,8 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const fs = require('fs') -const { Buffer } = require('buffer') const FormData = require('form-data') const streamToPromise = require('stream-to-promise') const { profiles } = require('../../../src/core/components/config') diff --git a/packages/ipfs/test/http-api/inject/dag.js b/packages/ipfs/test/http-api/inject/dag.js index 05214d75c4..181a202d45 100644 --- a/packages/ipfs/test/http-api/inject/dag.js +++ b/packages/ipfs/test/http-api/inject/dag.js @@ -2,11 +2,10 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const DAGNode = require('ipld-dag-pb').DAGNode const Readable = require('stream').Readable const FormData = require('form-data') -const { Buffer } = require('buffer') const streamToPromise = require('stream-to-promise') const CID = require('cids') const testHttpMethod = require('../../utils/test-http-method') @@ -72,7 +71,7 @@ describe('/dag', () => { }) it('returns value', async () => { - const node = new DAGNode(Buffer.from([]), []) + const node = new DAGNode(Uint8Array.from([]), []) ipfs.dag.get.withArgs(cid, defaultOptions).returns({ value: node }) const res = await http({ @@ -86,7 +85,7 @@ describe('/dag', () => { }) it('uses text encoding for data by default', async () => { - const node = new DAGNode(Buffer.from([0, 1, 2, 3]), []) + const node = new DAGNode(Uint8Array.from([0, 1, 2, 3]), []) ipfs.dag.get.withArgs(cid, defaultOptions).returns({ value: node }) const res = await http({ @@ -101,7 +100,7 @@ describe('/dag', () => { }) it('overrides data encoding', async () => { - const node = new DAGNode(Buffer.from([0, 1, 2, 3]), []) + const node = new DAGNode(Uint8Array.from([0, 1, 2, 3]), []) ipfs.dag.get.withArgs(cid, defaultOptions).returns({ value: node }) const res = await http({ @@ -130,7 +129,7 @@ describe('/dag', () => { }) it('returns value with a path as part of the cid for dag-pb nodes', async () => { - const node = new DAGNode(Buffer.from([0, 1, 2, 3]), []) + const node = new DAGNode(Uint8Array.from([0, 1, 2, 3]), []) ipfs.dag.get.withArgs(cid, { ...defaultOptions, path: '/Data' @@ -149,7 +148,7 @@ describe('/dag', () => { const node = { foo: 'bar', baz: { - qux: Buffer.from([0, 1, 2, 3]) + qux: Uint8Array.from([0, 1, 2, 3]) } } ipfs.dag.get.withArgs(cid, defaultOptions).returns({ value: node }) @@ -166,7 +165,7 @@ describe('/dag', () => { it('supports specifying buffer encoding', async () => { const node = { foo: 'bar', - baz: Buffer.from([0, 1, 2, 3]) + baz: Uint8Array.from([0, 1, 2, 3]) } ipfs.dag.get.withArgs(cid, defaultOptions).returns({ value: node }) diff --git a/packages/ipfs/test/http-api/inject/dht.js b/packages/ipfs/test/http-api/inject/dht.js index f4f59070dd..deb7640230 100644 --- a/packages/ipfs/test/http-api/inject/dht.js +++ b/packages/ipfs/test/http-api/inject/dht.js @@ -2,10 +2,9 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') -const { Buffer } = require('buffer') const sinon = require('sinon') const errCode = require('err-code') const CID = require('cids') diff --git a/packages/ipfs/test/http-api/inject/dns.js b/packages/ipfs/test/http-api/inject/dns.js index 55c175b766..f73108a810 100644 --- a/packages/ipfs/test/http-api/inject/dns.js +++ b/packages/ipfs/test/http-api/inject/dns.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const sinon = require('sinon') diff --git a/packages/ipfs/test/http-api/inject/files.js b/packages/ipfs/test/http-api/inject/files.js index c4fd21b274..d963ba8410 100644 --- a/packages/ipfs/test/http-api/inject/files.js +++ b/packages/ipfs/test/http-api/inject/files.js @@ -3,7 +3,7 @@ 'use strict' const randomBytes = require('iso-random-stream/src/random') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const FormData = require('form-data') const streamToPromise = require('stream-to-promise') const multibase = require('multibase') @@ -14,7 +14,6 @@ const CID = require('cids') const first = require('it-first') const toBuffer = require('it-to-buffer') const { AbortSignal } = require('abort-controller') -const { Buffer } = require('buffer') function matchIterable () { return sinon.match((thing) => Boolean(thing[Symbol.asyncIterator]) || Boolean(thing[Symbol.iterator])) diff --git a/packages/ipfs/test/http-api/inject/id.js b/packages/ipfs/test/http-api/inject/id.js index 1b773254c6..b4521c12d2 100644 --- a/packages/ipfs/test/http-api/inject/id.js +++ b/packages/ipfs/test/http-api/inject/id.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const sinon = require('sinon') diff --git a/packages/ipfs/test/http-api/inject/key.js b/packages/ipfs/test/http-api/inject/key.js index 7ae199b2e4..b98d33faac 100644 --- a/packages/ipfs/test/http-api/inject/key.js +++ b/packages/ipfs/test/http-api/inject/key.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const sinon = require('sinon') diff --git a/packages/ipfs/test/http-api/inject/mfs/chmod.js b/packages/ipfs/test/http-api/inject/mfs/chmod.js index 35982aa709..a3cb2feeb5 100644 --- a/packages/ipfs/test/http-api/inject/mfs/chmod.js +++ b/packages/ipfs/test/http-api/inject/mfs/chmod.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('../../../utils/http') const sinon = require('sinon') const testHttpMethod = require('../../../utils/test-http-method') diff --git a/packages/ipfs/test/http-api/inject/mfs/cp.js b/packages/ipfs/test/http-api/inject/mfs/cp.js index d99456a808..f2bbdbb2b0 100644 --- a/packages/ipfs/test/http-api/inject/mfs/cp.js +++ b/packages/ipfs/test/http-api/inject/mfs/cp.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('../../../utils/http') const sinon = require('sinon') const testHttpMethod = require('../../../utils/test-http-method') diff --git a/packages/ipfs/test/http-api/inject/mfs/flush.js b/packages/ipfs/test/http-api/inject/mfs/flush.js index 4a1d1f8095..f31eea8775 100644 --- a/packages/ipfs/test/http-api/inject/mfs/flush.js +++ b/packages/ipfs/test/http-api/inject/mfs/flush.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('../../../utils/http') const sinon = require('sinon') const CID = require('cids') diff --git a/packages/ipfs/test/http-api/inject/mfs/ls.js b/packages/ipfs/test/http-api/inject/mfs/ls.js index 5548b72e60..746142569e 100644 --- a/packages/ipfs/test/http-api/inject/mfs/ls.js +++ b/packages/ipfs/test/http-api/inject/mfs/ls.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('../../../utils/http') const sinon = require('sinon') const CID = require('cids') diff --git a/packages/ipfs/test/http-api/inject/mfs/mkdir.js b/packages/ipfs/test/http-api/inject/mfs/mkdir.js index 120a96cfbc..b51d339c84 100644 --- a/packages/ipfs/test/http-api/inject/mfs/mkdir.js +++ b/packages/ipfs/test/http-api/inject/mfs/mkdir.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('../../../utils/http') const sinon = require('sinon') const testHttpMethod = require('../../../utils/test-http-method') diff --git a/packages/ipfs/test/http-api/inject/mfs/mv.js b/packages/ipfs/test/http-api/inject/mfs/mv.js index d6239730ee..70dbd9a28f 100644 --- a/packages/ipfs/test/http-api/inject/mfs/mv.js +++ b/packages/ipfs/test/http-api/inject/mfs/mv.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('../../../utils/http') const sinon = require('sinon') const testHttpMethod = require('../../../utils/test-http-method') diff --git a/packages/ipfs/test/http-api/inject/mfs/read.js b/packages/ipfs/test/http-api/inject/mfs/read.js index 60edb178b7..28b06bf35d 100644 --- a/packages/ipfs/test/http-api/inject/mfs/read.js +++ b/packages/ipfs/test/http-api/inject/mfs/read.js @@ -1,9 +1,9 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('../../../utils/http') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const sinon = require('sinon') const testHttpMethod = require('../../../utils/test-http-method') const { AbortSignal } = require('abort-controller') @@ -22,7 +22,7 @@ describe('/files/read', () => { beforeEach(() => { ipfs = { files: { - read: sinon.stub().returns([Buffer.from('hello world')]) + read: sinon.stub().returns([uint8ArrayFromString('hello world')]) } } }) diff --git a/packages/ipfs/test/http-api/inject/mfs/rm.js b/packages/ipfs/test/http-api/inject/mfs/rm.js index 4fc0b0e96e..ea67171e0b 100644 --- a/packages/ipfs/test/http-api/inject/mfs/rm.js +++ b/packages/ipfs/test/http-api/inject/mfs/rm.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('../../../utils/http') const sinon = require('sinon') const testHttpMethod = require('../../../utils/test-http-method') diff --git a/packages/ipfs/test/http-api/inject/mfs/stat.js b/packages/ipfs/test/http-api/inject/mfs/stat.js index 67a788075c..8122fa2dfd 100644 --- a/packages/ipfs/test/http-api/inject/mfs/stat.js +++ b/packages/ipfs/test/http-api/inject/mfs/stat.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('../../../utils/http') const sinon = require('sinon') const CID = require('cids') diff --git a/packages/ipfs/test/http-api/inject/mfs/touch.js b/packages/ipfs/test/http-api/inject/mfs/touch.js index 4ed3520a6c..878510643c 100644 --- a/packages/ipfs/test/http-api/inject/mfs/touch.js +++ b/packages/ipfs/test/http-api/inject/mfs/touch.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('../../../utils/http') const sinon = require('sinon') const testHttpMethod = require('../../../utils/test-http-method') diff --git a/packages/ipfs/test/http-api/inject/mfs/write.js b/packages/ipfs/test/http-api/inject/mfs/write.js index 4d466a5187..feb1905ead 100644 --- a/packages/ipfs/test/http-api/inject/mfs/write.js +++ b/packages/ipfs/test/http-api/inject/mfs/write.js @@ -1,10 +1,9 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('../../../utils/http') const sinon = require('sinon') -const { Buffer } = require('buffer') const FormData = require('form-data') const streamToPromise = require('stream-to-promise') const { AbortSignal } = require('abort-controller') diff --git a/packages/ipfs/test/http-api/inject/name.js b/packages/ipfs/test/http-api/inject/name.js index 5d919ca126..2e726bc7b1 100644 --- a/packages/ipfs/test/http-api/inject/name.js +++ b/packages/ipfs/test/http-api/inject/name.js @@ -3,7 +3,7 @@ 'use strict' const CID = require('cids') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const sinon = require('sinon') diff --git a/packages/ipfs/test/http-api/inject/object.js b/packages/ipfs/test/http-api/inject/object.js index 5cacebeee8..244b6ab24e 100644 --- a/packages/ipfs/test/http-api/inject/object.js +++ b/packages/ipfs/test/http-api/inject/object.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const fs = require('fs') const FormData = require('form-data') const streamToPromise = require('stream-to-promise') @@ -11,7 +11,6 @@ const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const sinon = require('sinon') const CID = require('cids') -const { Buffer } = require('buffer') const UnixFS = require('ipfs-unixfs') const { AbortSignal } = require('abort-controller') const { @@ -555,7 +554,7 @@ describe('/object', () => { }, { ipfs }) expect(res).to.have.property('statusCode', 200) - expect(res).to.have.property('result', emptyDirectoryNode.Data.toString()) + expect(res).to.have.property('result', emptyDirectoryNode.Data) }) it('accepts a timeout', async () => { @@ -570,7 +569,7 @@ describe('/object', () => { }, { ipfs }) expect(res).to.have.property('statusCode', 200) - expect(res).to.have.property('result', emptyDirectoryNode.Data.toString()) + expect(res).to.have.property('result', emptyDirectoryNode.Data) }) }) diff --git a/packages/ipfs/test/http-api/inject/pin.js b/packages/ipfs/test/http-api/inject/pin.js index de93744863..83ff63f8a1 100644 --- a/packages/ipfs/test/http-api/inject/pin.js +++ b/packages/ipfs/test/http-api/inject/pin.js @@ -2,7 +2,7 @@ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const multibase = require('multibase') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') diff --git a/packages/ipfs/test/http-api/inject/ping.js b/packages/ipfs/test/http-api/inject/ping.js index f677bd06d6..922a06b0f7 100644 --- a/packages/ipfs/test/http-api/inject/ping.js +++ b/packages/ipfs/test/http-api/inject/ping.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const sinon = require('sinon') diff --git a/packages/ipfs/test/http-api/inject/pubsub.js b/packages/ipfs/test/http-api/inject/pubsub.js index 4b15d5ef66..95374d0fa9 100644 --- a/packages/ipfs/test/http-api/inject/pubsub.js +++ b/packages/ipfs/test/http-api/inject/pubsub.js @@ -2,14 +2,13 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const FormData = require('form-data') const sinon = require('sinon') const { AbortSignal } = require('abort-controller') const randomBytes = require('iso-random-stream/src/random') -const { Buffer } = require('buffer') const streamToPromise = require('stream-to-promise') const sendData = async (data) => { diff --git a/packages/ipfs/test/http-api/inject/repo.js b/packages/ipfs/test/http-api/inject/repo.js index 692ed0c76e..c6eb91bf46 100644 --- a/packages/ipfs/test/http-api/inject/repo.js +++ b/packages/ipfs/test/http-api/inject/repo.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const sinon = require('sinon') diff --git a/packages/ipfs/test/http-api/inject/resolve.js b/packages/ipfs/test/http-api/inject/resolve.js index d5d4130bdd..3d691c7249 100644 --- a/packages/ipfs/test/http-api/inject/resolve.js +++ b/packages/ipfs/test/http-api/inject/resolve.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const sinon = require('sinon') diff --git a/packages/ipfs/test/http-api/inject/stats.js b/packages/ipfs/test/http-api/inject/stats.js index 3fa968ad7b..5f1c779896 100644 --- a/packages/ipfs/test/http-api/inject/stats.js +++ b/packages/ipfs/test/http-api/inject/stats.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const sinon = require('sinon') diff --git a/packages/ipfs/test/http-api/inject/swarm.js b/packages/ipfs/test/http-api/inject/swarm.js index d864ebe0fd..547092ab5a 100644 --- a/packages/ipfs/test/http-api/inject/swarm.js +++ b/packages/ipfs/test/http-api/inject/swarm.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const sinon = require('sinon') diff --git a/packages/ipfs/test/http-api/inject/version.js b/packages/ipfs/test/http-api/inject/version.js index 10b0afa140..d92cf76333 100644 --- a/packages/ipfs/test/http-api/inject/version.js +++ b/packages/ipfs/test/http-api/inject/version.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') const sinon = require('sinon') diff --git a/packages/ipfs/test/utils/test-http-method.js b/packages/ipfs/test/utils/test-http-method.js index 92a8f7a07e..7d7ae98f47 100644 --- a/packages/ipfs/test/utils/test-http-method.js +++ b/packages/ipfs/test/utils/test-http-method.js @@ -1,6 +1,6 @@ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const http = require('./http') const METHODS = [ From 0d3a1a146082bdffc261f85264702954e07f951a Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 15 Aug 2020 09:27:52 +0100 Subject: [PATCH 22/40] chore: add missing dep --- packages/ipfs-core-utils/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 496ebab592..4755731985 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -35,7 +35,8 @@ "ipfs-utils": "ipfs/js-ipfs-utils#revert-54-xhr", "it-all": "^1.0.1", "it-map": "^1.0.0", - "it-peekable": "0.0.1" + "it-peekable": "0.0.1", + "uint8arrays": "^1.1.0" }, "devDependencies": { "aegir": "^26.0.0", From aa591c7233b740f5a07f7fc04cf6fe572cb17856 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 15 Aug 2020 09:57:52 +0100 Subject: [PATCH 23/40] chore: add missing deps --- packages/ipfs-core-utils/package.json | 2 +- packages/ipfs-http-client/package.json | 1 + packages/ipfs-message-port-protocol/package.json | 3 ++- packages/ipfs/package.json | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 4755731985..ff2ab615d3 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -34,7 +34,7 @@ "err-code": "^2.0.0", "ipfs-utils": "ipfs/js-ipfs-utils#revert-54-xhr", "it-all": "^1.0.1", - "it-map": "^1.0.0", + "it-map": "^1.0.2", "it-peekable": "0.0.1", "uint8arrays": "^1.1.0" }, diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 418f4f4313..839b9e1661 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -54,6 +54,7 @@ "ipld-raw": "^6.0.0", "iso-url": "^0.4.7", "it-last": "^1.0.1", + "it-map": "^1.0.2", "it-tar": "^1.2.2", "it-to-buffer": "^1.0.0", "it-to-stream": "^0.1.1", diff --git a/packages/ipfs-message-port-protocol/package.json b/packages/ipfs-message-port-protocol/package.json index 6d41c91d0a..014d2fb422 100644 --- a/packages/ipfs-message-port-protocol/package.json +++ b/packages/ipfs-message-port-protocol/package.json @@ -37,7 +37,8 @@ }, "devDependencies": { "aegir": "^26.0.0", - "interface-ipfs-core": "^0.139.0" + "interface-ipfs-core": "^0.139.0", + "uint8arrays": "^1.1.0" }, "engines": { "node": ">=10.3.0", diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 6931b015cf..07328090ca 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -121,7 +121,7 @@ "it-first": "^1.0.1", "it-glob": "0.0.8", "it-last": "^1.0.1", - "it-map": "^1.0.0", + "it-map": "^1.0.2", "it-multipart": "^1.0.1", "it-pipe": "^1.1.0", "it-tar": "^1.2.2", From 86bcd707c3227025031dad6c5682da02314e4eae Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 15 Aug 2020 11:06:11 +0100 Subject: [PATCH 24/40] chore: fix up normalisation tests --- .../files/normalise-input/normalise-content.browser.js | 3 ++- .../src/files/normalise-input/normalise-content.js | 10 +++++++++- .../ipfs-core-utils/test/files/normalise-input.spec.js | 3 ++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js index fe55cde87c..415ec8dd86 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js @@ -4,6 +4,7 @@ const errCode = require('err-code') const { Blob } = require('ipfs-utils/src/globalthis') const itPeekable = require('it-peekable') const browserStreamToIt = require('browser-readablestream-to-it') +const all = require('it-all') const { isBytes, @@ -40,7 +41,7 @@ async function toBlob (input) { // (Async)Iterable if (Number.isInteger(value)) { - return itToBlob(peekable) + return new Blob([Uint8Array.from(await all(peekable))]) } // (Async)Iterable diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js index 6144717350..1e726ca49a 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js @@ -60,7 +60,15 @@ async function * toAsyncIterable (input) { } function toBuffer (chunk) { - return isBytes(chunk) ? chunk : uint8ArrayFromString(chunk) + if (isBytes(chunk)) { + return chunk + } + + if (Array.isArray(chunk)) { + return Uint8Array.from(chunk) + } + + return uint8ArrayFromString(chunk) } module.exports = toAsyncIterable diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index 383f327c99..203d01a520 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -36,10 +36,11 @@ async function verifyNormalisation (input) { if (isBrowser || isWebWorker) { expect(content).to.be.an.instanceOf(Blob) - content = blobToIt(input[0].content) + content = blobToIt(content) } expect(content[Symbol.asyncIterator] || content[Symbol.iterator]).to.be.ok('Content should have been an iterable or an async iterable') + await expect(all(content)).to.eventually.deep.equal([BUFFER()]) } From cdd80ef19f39d54b681d2e3e858edec886e884e0 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 15 Aug 2020 11:33:13 +0100 Subject: [PATCH 25/40] chore: fix typo --- packages/ipfs/src/core/components/init.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs/src/core/components/init.js b/packages/ipfs/src/core/components/init.js index e6adadb49c..e8c4040725 100644 --- a/packages/ipfs/src/core/components/init.js +++ b/packages/ipfs/src/core/components/init.js @@ -3,7 +3,7 @@ const log = require('debug')('ipfs:components:init') const PeerId = require('peer-id') const uint8ArrayFromString = require('uint8arrays/from-string') -const uint8ArrayToString = require() +const uint8ArrayToString = require('uint8arrays/to-string') const mergeOptions = require('merge-options') const getDefaultConfig = require('../runtime/config-nodejs.js') From bfc74d6361680d212311b1cab456692d27394a8b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 15 Aug 2020 12:08:06 +0100 Subject: [PATCH 26/40] chore: fix failing test --- packages/ipfs-message-port-protocol/src/cid.js | 2 +- .../test/dag.browser.js | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/ipfs-message-port-protocol/src/cid.js b/packages/ipfs-message-port-protocol/src/cid.js index a79895301b..c142ead8a3 100644 --- a/packages/ipfs-message-port-protocol/src/cid.js +++ b/packages/ipfs-message-port-protocol/src/cid.js @@ -34,7 +34,7 @@ exports.encodeCID = encodeCID const decodeCID = encodedCID => { /** @type {CID} */ const cid = (encodedCID) - Object.setPrototypeOf(cid.multihash, Buffer.prototype) + Object.setPrototypeOf(cid.multihash, Uint8Array.prototype) Object.setPrototypeOf(cid, CID.prototype) // TODO: Figure out a way to avoid `Symbol.for` here as it can get out of // sync with cids implementation. diff --git a/packages/ipfs-message-port-protocol/test/dag.browser.js b/packages/ipfs-message-port-protocol/test/dag.browser.js index 34451b858f..d70805284b 100644 --- a/packages/ipfs-message-port-protocol/test/dag.browser.js +++ b/packages/ipfs-message-port-protocol/test/dag.browser.js @@ -40,10 +40,8 @@ describe('dag (browser)', function () { }) it('should decode dagNode over message channel & transfer bytes', async () => { - const cid1 = new CID( - 'bafyreic6f672hnponukaacmk2mmt7vs324zkagvu4hcww6yba6kby25zce' - ) - const cid2 = new CID('QmPv52ekjS75L4JmHpXVeuJ5uX2ecSfSZo88NSyxwA3rAQ') + const cid1 = 'bafyreic6f672hnponukaacmk2mmt7vs324zkagvu4hcww6yba6kby25zce' + const cid2 = 'QmPv52ekjS75L4JmHpXVeuJ5uX2ecSfSZo88NSyxwA3rAQ' const hi = uint8ArrayFromString('hello world') const nodeIn = { @@ -51,7 +49,9 @@ describe('dag (browser)', function () { nested: { structure: { with: { - links: [new CID(cid1)] + links: [ + new CID(cid1) + ] } } }, @@ -66,16 +66,18 @@ describe('dag (browser)', function () { ) expect(nodeOut).to.be.deep.equal({ - hi, + hi: uint8ArrayFromString('hello world'), nested: { structure: { with: { - links: [cid1] + links: [ + new CID(cid1) + ] } } }, other: { - link: cid2 + link: new CID(cid2) } }) From 173905ff693d596e4f1f6c8cfbe9598cfd1ccb87 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 15 Aug 2020 12:20:35 +0100 Subject: [PATCH 27/40] chore: fix pubsub tests --- packages/ipfs/src/http/api/resources/pubsub.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/ipfs/src/http/api/resources/pubsub.js b/packages/ipfs/src/http/api/resources/pubsub.js index 4ae6c2ca56..1910f81b59 100644 --- a/packages/ipfs/src/http/api/resources/pubsub.js +++ b/packages/ipfs/src/http/api/resources/pubsub.js @@ -6,6 +6,8 @@ const bs58 = require('bs58') const all = require('it-all') const multipart = require('../../utils/multipart-request-parser') const Boom = require('@hapi/boom') +const uint8ArrayToString = require('uint8arrays/to-string') +const uint8ArrayFromString = require('uint8arrays/from-string') exports.subscribe = { options: { @@ -46,9 +48,9 @@ exports.subscribe = { const handler = (msg) => { res.write(JSON.stringify({ - from: bs58.decode(msg.from).toString('base64'), - data: msg.data.toString('base64'), - seqno: msg.seqno.toString('base64'), + from: uint8ArrayToString(uint8ArrayFromString(msg.from, 'base58btc'), 'base64pad'), + data: uint8ArrayToString(msg.data, 'base64pad'), + seqno: uint8ArrayToString(msg.seqno, 'base64pad'), topicIDs: msg.topicIDs }) + '\n', 'utf8') } From d072a407f53bc733d2cb883b06b93e3fc0b07c65 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 15 Aug 2020 12:21:11 +0100 Subject: [PATCH 28/40] chore: remove unused dep --- packages/ipfs/package.json | 1 - packages/ipfs/src/http/api/resources/pubsub.js | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 07328090ca..4163fe39d3 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -72,7 +72,6 @@ "array-shuffle": "^1.0.1", "bignumber.js": "^9.0.0", "bl": "^4.0.2", - "bs58": "^4.0.1", "byteman": "^1.3.5", "cbor": "^5.0.1", "cid-tool": "multiformats/js-cid-tool#fix/replace-node-buffers-with-uint8arrays", diff --git a/packages/ipfs/src/http/api/resources/pubsub.js b/packages/ipfs/src/http/api/resources/pubsub.js index 1910f81b59..2d2d8685c9 100644 --- a/packages/ipfs/src/http/api/resources/pubsub.js +++ b/packages/ipfs/src/http/api/resources/pubsub.js @@ -2,7 +2,6 @@ const Joi = require('../../utils/joi') const PassThrough = require('stream').PassThrough -const bs58 = require('bs58') const all = require('it-all') const multipart = require('../../utils/multipart-request-parser') const Boom = require('@hapi/boom') From 5879a492587a5b53d11c86160bbe44a81279beed Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 15 Aug 2020 13:43:58 +0100 Subject: [PATCH 29/40] chore: fix failing test --- packages/ipfs/test/core/utils.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ipfs/test/core/utils.js b/packages/ipfs/test/core/utils.js index 7c657b02af..f658da74e1 100644 --- a/packages/ipfs/test/core/utils.js +++ b/packages/ipfs/test/core/utils.js @@ -52,20 +52,20 @@ describe('utils', () => { it('handles base58 hash format', async () => { const hash = await utils.resolvePath(node.dag, rootHash) - expect(hash).to.have.property('buffer').that.deep.equals(rootMultihash) + expect(hash).to.have.property('bytes').that.deep.equals(rootMultihash) }) it('handles multihash format', async () => { const hash = await utils.resolvePath(node.dag, aboutMultihash) - expect(hash).to.have.property('buffer').that.deep.equals(aboutMultihash) + expect(hash).to.have.property('bytes').that.deep.equals(aboutMultihash) }) it('handles ipfs paths format', async function () { this.timeout(200 * 1000) const hash = await utils.resolvePath(node.dag, aboutPath) - expect(hash).to.have.property('buffer').that.deep.equals(aboutMultihash) + expect(hash).to.have.property('bytes').that.deep.equals(aboutMultihash) }) it('should error on invalid hashes', () => { From caf8ba5cd94a42e3b3ccdd9a0ed80be5b6bb0062 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 15 Aug 2020 15:46:30 +0100 Subject: [PATCH 30/40] chore: use base64pad and print migration progress --- examples/custom-ipfs-repo/package.json | 2 +- packages/ipfs/package.json | 2 +- packages/ipfs/src/cli/commands/object/get.js | 12 +++++++++++ packages/ipfs/src/cli/daemon.js | 2 +- packages/ipfs/src/core/components/init.js | 4 ++-- packages/ipfs/src/core/runtime/repo-nodejs.js | 21 ++++++++++++++++++- .../ipfs/src/http/api/resources/object.js | 9 +++++--- 7 files changed, 43 insertions(+), 9 deletions(-) diff --git a/examples/custom-ipfs-repo/package.json b/examples/custom-ipfs-repo/package.json index 2526e36527..645b9e3d6b 100644 --- a/examples/custom-ipfs-repo/package.json +++ b/examples/custom-ipfs-repo/package.json @@ -12,7 +12,7 @@ "dependencies": { "datastore-fs": "^2.0.0", "ipfs": "^0.49.0", - "ipfs-repo": "^6.0.1", + "ipfs-repo": "^6.0.3", "it-all": "^1.0.1" }, "devDependencies": { diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 4163fe39d3..9b1a619135 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -96,7 +96,7 @@ "ipfs-core-utils": "^0.3.1", "ipfs-http-client": "^46.0.0", "ipfs-http-response": "^0.6.0", - "ipfs-repo": "^6.0.1", + "ipfs-repo": "^6.0.3", "ipfs-unixfs": "^2.0.2", "ipfs-unixfs-exporter": "^3.0.2", "ipfs-unixfs-importer": "^3.0.2", diff --git a/packages/ipfs/src/cli/commands/object/get.js b/packages/ipfs/src/cli/commands/object/get.js index c9618d0055..b241daebe0 100644 --- a/packages/ipfs/src/cli/commands/object/get.js +++ b/packages/ipfs/src/cli/commands/object/get.js @@ -30,6 +30,18 @@ module.exports = { const node = await ipfs.object.get(key, { enc: 'base58', timeout }) let data = node.Data || '' + if (dataEncoding === 'base64') { + dataEncoding = 'base64pad' + } + + if (dataEncoding === 'text') { + dataEncoding = 'ascii' + } + + if (dataEncoding === 'hex') { + dataEncoding = 'base16' + } + if (data instanceof Uint8Array) { data = uint8ArrayToString(node.Data, dataEncoding || undefined) } diff --git a/packages/ipfs/src/cli/daemon.js b/packages/ipfs/src/cli/daemon.js index e16d58999e..8dc21a7f3a 100644 --- a/packages/ipfs/src/cli/daemon.js +++ b/packages/ipfs/src/cli/daemon.js @@ -29,7 +29,7 @@ class Daemon { log('starting') const repo = typeof this._options.repo === 'string' || this._options.repo == null - ? createRepo({ path: this._options.repo, autoMigrate: this._options.repoAutoMigrate }) + ? createRepo({ path: this._options.repo, autoMigrate: this._options.repoAutoMigrate, silent: this._options.silent }) : this._options.repo // start the daemon diff --git a/packages/ipfs/src/core/components/init.js b/packages/ipfs/src/core/components/init.js index e8c4040725..70f589143e 100644 --- a/packages/ipfs/src/core/components/init.js +++ b/packages/ipfs/src/core/components/init.js @@ -53,7 +53,7 @@ module.exports = ({ options.repoAutoMigrate = options.repoAutoMigrate || constructorOptions.repoAutoMigrate const repo = typeof options.repo === 'string' || options.repo == null - ? createRepo({ path: options.repo, autoMigrate: options.repoAutoMigrate }) + ? createRepo({ path: options.repo, autoMigrate: options.repoAutoMigrate, silent: constructorOptions.silent }) : options.repo let isInitialized = true @@ -266,7 +266,7 @@ function createPeerId ({ privateKey, algorithm = 'rsa', bits, print }) { log('using user-supplied private-key') return typeof privateKey === 'object' ? privateKey - : PeerId.createFromPrivKey(uint8ArrayFromString(privateKey, 'base64')) + : PeerId.createFromPrivKey(uint8ArrayFromString(privateKey, 'base64pad')) } else { // Generate peer identity keypair + transform to desired format + add to config. print('generating %s-bit (rsa only) %s keypair...', bits, algorithm) diff --git a/packages/ipfs/src/core/runtime/repo-nodejs.js b/packages/ipfs/src/core/runtime/repo-nodejs.js index d8581b7e32..c0208f7582 100644 --- a/packages/ipfs/src/core/runtime/repo-nodejs.js +++ b/packages/ipfs/src/core/runtime/repo-nodejs.js @@ -7,5 +7,24 @@ const path = require('path') module.exports = options => { options = options || {} const repoPath = options.path || path.join(os.homedir(), '.jsipfs') - return new IPFSRepo(repoPath, { autoMigrate: options.autoMigrate }) + let lastMigration = null + + let onMigrationProgress = (version, percentComplete, message) => { + if (version !== lastMigration) { + lastMigration = version + + console.info(`Migrating repo from v${version - 1} to v${version}`) // eslint-disable-line no-console + } + + console.info(`${percentComplete.toString().padStart(6, ' ')}% ${message}`) // eslint-disable-line no-console + } + + if (options.silent) { + onMigrationProgress = null + } + + return new IPFSRepo(repoPath, { + autoMigrate: options.autoMigrate, + onMigrationProgress + }) } diff --git a/packages/ipfs/src/http/api/resources/object.js b/packages/ipfs/src/http/api/resources/object.js index 98904bc749..ff274d715a 100644 --- a/packages/ipfs/src/http/api/resources/object.js +++ b/packages/ipfs/src/http/api/resources/object.js @@ -7,6 +7,7 @@ const { DAGLink } = dagPB const Joi = require('../../utils/joi') const multibase = require('multibase') const Boom = require('@hapi/boom') +const uint8ArrayToString = require('uint8arrays/to-string') const { cidToString } = require('../../../utils/cid') const debug = require('debug') const log = debug('ipfs:http-api:object') @@ -130,8 +131,10 @@ exports.get = { cidBase: Joi.cidBase(), enc: Joi.string(), dataEncoding: Joi.string() - .valid('text', 'base64', 'hex', 'utf8') - .replace(/text/, 'utf8') + .valid('ascii', 'base64pad', 'base16', 'utf8') + .replace(/text/, 'ascii') + .replace(/base64/, 'base64pad') + .replace(/hex/, 'base16') .default('utf8'), timeout: Joi.timeout() }) @@ -180,7 +183,7 @@ exports.get = { } return h.response({ - Data: node.Data.toString(dataEncoding), + Data: uint8ArrayToString(node.Data, dataEncoding), Hash: cidToString(cid, { base: cidBase, upgrade: false }), Size: node.size, Links: node.Links.map((l) => { From 50ad0f208f81733e5c628d2e3efb5c75f024db00 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 15 Aug 2020 17:26:31 +0100 Subject: [PATCH 31/40] chore: fix object data --- packages/ipfs/src/http/api/resources/object.js | 4 ++-- packages/ipfs/test/http-api/inject/object.js | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/ipfs/src/http/api/resources/object.js b/packages/ipfs/src/http/api/resources/object.js index ff274d715a..c3bb5c725a 100644 --- a/packages/ipfs/src/http/api/resources/object.js +++ b/packages/ipfs/src/http/api/resources/object.js @@ -103,7 +103,7 @@ exports.new = { const nodeJSON = node.toJSON() const answer = { - Data: nodeJSON.data, + Data: uint8ArrayToString(node.Data, 'base64pad'), Hash: cidToString(cid, { base: cidBase, upgrade: false }), Size: nodeJSON.size, Links: nodeJSON.links.map((l) => { @@ -135,7 +135,7 @@ exports.get = { .replace(/text/, 'ascii') .replace(/base64/, 'base64pad') .replace(/hex/, 'base16') - .default('utf8'), + .default('base64pad'), timeout: Joi.timeout() }) .rename('cid-base', 'cidBase', { diff --git a/packages/ipfs/test/http-api/inject/object.js b/packages/ipfs/test/http-api/inject/object.js index 244b6ab24e..760441c4ef 100644 --- a/packages/ipfs/test/http-api/inject/object.js +++ b/packages/ipfs/test/http-api/inject/object.js @@ -17,6 +17,7 @@ const { DAGNode, DAGLink } = require('ipld-dag-pb') +const uint8ArrayToString = require('uint8arrays/to-string') describe('/object', () => { const cid = new CID('QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n') @@ -205,7 +206,7 @@ describe('/object', () => { expect(res).to.have.property('statusCode', 200) expect(res).to.have.nested.property('result.Links').that.is.empty() - expect(res).to.have.nested.property('result.Data', emptyDirectoryNode.Data.toString()) + expect(res).to.have.nested.property('result.Data', uint8ArrayToString(emptyDirectoryNode.Data, 'base64pad')) }) // TODO: unskip after switch to v1 CIDs by default @@ -244,7 +245,7 @@ describe('/object', () => { expect(res).to.have.property('statusCode', 200) expect(res).to.have.nested.property('result.Links').that.is.empty() - expect(res).to.have.nested.property('result.Data', emptyDirectoryNode.Data.toString()) + expect(res).to.have.nested.property('result.Data', uint8ArrayToString(emptyDirectoryNode.Data, 'base64pad')) }) }) From 903e2cf966a1697e1b27d9325a6d29283ce10e2f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 15 Aug 2020 18:13:36 +0100 Subject: [PATCH 32/40] chore: fix base string --- packages/ipfs/test/core/init.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs/test/core/init.spec.js b/packages/ipfs/test/core/init.spec.js index b7a92b83cb..9daba6024e 100644 --- a/packages/ipfs/test/core/init.spec.js +++ b/packages/ipfs/test/core/init.spec.js @@ -120,7 +120,7 @@ describe('init', function () { await ipfs.init({ bits: 512, emptyRepo: true }) // Should not have default assets - const multihash = uint8ArrayFromString('12205e7c3ce237f936c76faf625e90f7751a9f5eeb048f59873303c215e9cce87599', 'hex') + const multihash = uint8ArrayFromString('12205e7c3ce237f936c76faf625e90f7751a9f5eeb048f59873303c215e9cce87599', 'base16') await expect(ipfs.object.get(multihash, {})).to.eventually.be.rejected() }) From ba5c29bd03d3d5f0c13ba7de36119d2af8660dd4 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 16 Aug 2020 09:50:53 +0100 Subject: [PATCH 33/40] chore: fix failing tests --- examples/browser-exchange-files/package.json | 4 ++-- examples/browser-exchange-files/test.js | 1 + examples/ipfs-101/1.js | 1 + examples/ipfs-101/package.json | 3 ++- examples/test-ipfs-example/package.json | 3 ++- packages/ipfs-core-utils/package.json | 4 ++-- packages/ipfs-message-port-client/package.json | 2 +- packages/ipfs-message-port-client/src/core.js | 2 ++ packages/ipfs-message-port-client/test/interface.spec.js | 4 ++-- 9 files changed, 15 insertions(+), 9 deletions(-) diff --git a/examples/browser-exchange-files/package.json b/examples/browser-exchange-files/package.json index 9f7db3bb31..a6f4ac6ea7 100644 --- a/examples/browser-exchange-files/package.json +++ b/examples/browser-exchange-files/package.json @@ -14,12 +14,12 @@ "browserify": "^16.2.3", "execa": "^4.0.0", "http-server": "^0.11.1", - "ipfs-http-client": "^46.0.0" + "ipfs-http-client": "^46.0.0", + "uint8arrays": "^1.1.0" }, "dependencies": { "ipfs": "^0.49.0", "it-all": "^1.0.1", - "it-last": "^1.0.1", "test-ipfs-example": "^2.0.3" }, "browser": { diff --git a/examples/browser-exchange-files/test.js b/examples/browser-exchange-files/test.js index 208b3448fe..a7fd97b03b 100644 --- a/examples/browser-exchange-files/test.js +++ b/examples/browser-exchange-files/test.js @@ -20,6 +20,7 @@ const { } = require('test-ipfs-example/utils') const pkg = require('./package.json') const webRTCStarSigServer = require('libp2p-webrtc-star/src/sig-server') +const uint8ArrayToString = require('uint8arrays/to-string') const FILE_CONTENT = 'A file with some content' diff --git a/examples/ipfs-101/1.js b/examples/ipfs-101/1.js index a277e7096b..9f78faf8ae 100644 --- a/examples/ipfs-101/1.js +++ b/examples/ipfs-101/1.js @@ -3,6 +3,7 @@ const IPFS = require('ipfs') const all = require('it-all') const uint8ArrayConcat = require('uint8arrays/concat') +const uint8ArrayFromString = require('uint8arrays/from-string') async function main () { const node = await IPFS.create() diff --git a/examples/ipfs-101/package.json b/examples/ipfs-101/package.json index 4597603934..34612e0262 100644 --- a/examples/ipfs-101/package.json +++ b/examples/ipfs-101/package.json @@ -11,7 +11,8 @@ "license": "MIT", "dependencies": { "ipfs": "^0.49.0", - "it-all": "^1.0.1" + "it-all": "^1.0.1", + "uint8arrays": "^1.1.0" }, "devDependencies": { "test-ipfs-example": "^2.0.3" diff --git a/examples/test-ipfs-example/package.json b/examples/test-ipfs-example/package.json index 4f5590c846..db84f66784 100644 --- a/examples/test-ipfs-example/package.json +++ b/examples/test-ipfs-example/package.json @@ -12,6 +12,7 @@ "fs-extra": "^8.1.0", "http-server": "^0.11.1", "nightwatch": "^1.2.4", - "which": "^2.0.1" + "which": "^2.0.1", + "uint8arrays": "^1.1.0" } } diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index ff2ab615d3..dc6b23b5b4 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -28,8 +28,8 @@ }, "license": "MIT", "dependencies": { - "blob-to-it": "0.0.1", - "browser-readablestream-to-it": "0.0.1", + "blob-to-it": "0.0.2", + "browser-readablestream-to-it": "0.0.2", "cids": "^1.0.0", "err-code": "^2.0.0", "ipfs-utils": "ipfs/js-ipfs-utils#revert-54-xhr", diff --git a/packages/ipfs-message-port-client/package.json b/packages/ipfs-message-port-client/package.json index e1ca16a442..521759aea2 100644 --- a/packages/ipfs-message-port-client/package.json +++ b/packages/ipfs-message-port-client/package.json @@ -34,7 +34,7 @@ "dep-check": "aegir dep-check" }, "dependencies": { - "browser-readablestream-to-it": "0.0.1", + "browser-readablestream-to-it": "0.0.2", "cids": "^1.0.0" }, "devDependencies": { diff --git a/packages/ipfs-message-port-client/src/core.js b/packages/ipfs-message-port-client/src/core.js index 603a63f168..71ef34c924 100644 --- a/packages/ipfs-message-port-client/src/core.js +++ b/packages/ipfs-message-port-client/src/core.js @@ -367,6 +367,8 @@ const encodeFileContent = (content, transfer) => { return content } else if (content instanceof Blob) { return content + } else if (typeof content === 'string') { + return content } else { const iterable = asIterable(content) if (iterable) { diff --git a/packages/ipfs-message-port-client/test/interface.spec.js b/packages/ipfs-message-port-client/test/interface.spec.js index 4d104983cd..5be89147b8 100644 --- a/packages/ipfs-message-port-client/test/interface.spec.js +++ b/packages/ipfs-message-port-client/test/interface.spec.js @@ -65,8 +65,8 @@ describe('interface-ipfs-core tests', () => { reason: 'ipfs.object.get is not implemented' }, { - name: 'should cat with a Buffer multihash', - reason: 'Passing CID as Buffer is not supported' + name: 'should cat with a Uint8Array multihash', + reason: 'Passing CID as Uint8Array is not supported' }, { name: 'should add from a HTTP URL', From 5ad090628f1674b0927ca7b2e57b3d2a8dc390ee Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 16 Aug 2020 10:29:16 +0100 Subject: [PATCH 34/40] chore: fix up examples --- examples/browser-exchange-files/test.js | 3 +-- examples/browser-parceljs/public/index.js | 3 ++- examples/http-client-browser-pubsub/index.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/browser-exchange-files/test.js b/examples/browser-exchange-files/test.js index a7fd97b03b..6088bc24b4 100644 --- a/examples/browser-exchange-files/test.js +++ b/examples/browser-exchange-files/test.js @@ -20,7 +20,6 @@ const { } = require('test-ipfs-example/utils') const pkg = require('./package.json') const webRTCStarSigServer = require('libp2p-webrtc-star/src/sig-server') -const uint8ArrayToString = require('uint8arrays/to-string') const FILE_CONTENT = 'A file with some content' @@ -195,7 +194,7 @@ module.exports[pkg.name] = function (browser) { .then(res => res.text()) .then(done, done) }, [ url ], ({ value: contents }) => { - browser.expect(uint8ArrayToString(contents, 'utf8')).to.equal(FILE_CONTENT) + browser.expect(contents).to.equal(FILE_CONTENT) }) }) } diff --git a/examples/browser-parceljs/public/index.js b/examples/browser-parceljs/public/index.js index 7010fb7848..787d012737 100644 --- a/examples/browser-parceljs/public/index.js +++ b/examples/browser-parceljs/public/index.js @@ -1,6 +1,7 @@ import 'babel-polyfill' import IPFS from 'ipfs' import uint8ArrayConcat from 'uint8arrays/concat' +import uint8ArrayToString from 'uint8arrays/to-string' document.addEventListener('DOMContentLoaded', async () => { // IPFS node setup @@ -35,5 +36,5 @@ document.addEventListener('DOMContentLoaded', async () => { buffers.push(buf) } - log(`The contents of the file was: ${uint8ArrayConcat(buffers).toString()}`) + log(`The contents of the file was: ${uint8ArrayToString(uint8ArrayConcat(buffers))}`) }) diff --git a/examples/http-client-browser-pubsub/index.js b/examples/http-client-browser-pubsub/index.js index 0270734b41..3a21137a7a 100644 --- a/examples/http-client-browser-pubsub/index.js +++ b/examples/http-client-browser-pubsub/index.js @@ -81,7 +81,7 @@ async function main () { if (from === peerId) return log(`Ignoring message ${seqno} from self`) log(`Message ${seqno} from ${from}:`) try { - log(JSON.stringify(msg.data.toString(), null, 2)) + log(JSON.stringify(uint8ArrayToString(msg.data), null, 2)) } catch (_) { log(uint8ArrayToString(msg.data, 'base16')) } From a73ec06608c7ccd73a902b102e4bc37376f512da Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 16 Aug 2020 10:57:03 +0100 Subject: [PATCH 35/40] chore: fix circuit relay test --- examples/circuit-relaying/package.json | 3 ++- examples/circuit-relaying/src/helpers.js | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/circuit-relaying/package.json b/examples/circuit-relaying/package.json index 737396af15..79d4780f99 100644 --- a/examples/circuit-relaying/package.json +++ b/examples/circuit-relaying/package.json @@ -16,7 +16,8 @@ "dependencies": { "delay": "^4.3.0", "ipfs": "^0.49.0", - "ipfs-pubsub-room": "^2.0.1" + "ipfs-pubsub-room": "^2.0.1", + "uint8arrays": "^1.1.0" }, "devDependencies": { "execa": "^4.0.0", diff --git a/examples/circuit-relaying/src/helpers.js b/examples/circuit-relaying/src/helpers.js index 255b9be2c0..705ed2eca8 100644 --- a/examples/circuit-relaying/src/helpers.js +++ b/examples/circuit-relaying/src/helpers.js @@ -1,6 +1,8 @@ /* eslint-disable no-console */ 'use strict' +const uint8ArrayToString = require('uint8arrays/to-string') + const Room = require('ipfs-pubsub-room') const $message = document.querySelector('#message') const $msgs = document.querySelector('#msgs') @@ -32,9 +34,9 @@ module.exports = (ipfs, peersSet) => { // send and receive messages room.on('message', (message) => { - console.log('got message from ' + message.from + ': ' + message.data.toString()) + console.log('got message from ' + message.from + ': ' + uint8ArrayToString(message.data)) const node = document.createElement('li') - node.innerText = `${message.from.substr(-4)}: ${message.data.toString()}` + node.innerText = `${message.from.substr(-4)}: ${uint8ArrayToString(message.data)}` $msgs.appendChild(node) }) From a069f4c49296984a36f925b94fa6ce865d9af33c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 17 Aug 2020 07:47:11 +0100 Subject: [PATCH 36/40] chore: ignore invalid messages --- packages/ipfs-http-client/src/pubsub/subscribe.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/ipfs-http-client/src/pubsub/subscribe.js b/packages/ipfs-http-client/src/pubsub/subscribe.js index 9cd1215507..c62fc8c8cf 100644 --- a/packages/ipfs-http-client/src/pubsub/subscribe.js +++ b/packages/ipfs-http-client/src/pubsub/subscribe.js @@ -59,6 +59,10 @@ async function readMessages (msgStream, { onMessage, onEnd, onError }) { try { for await (const msg of msgStream) { try { + if (!msg.from) { + continue + } + onMessage({ from: uint8ArrayToString(uint8ArrayFromString(msg.from, 'base64pad'), 'base58btc'), data: uint8ArrayFromString(msg.data, 'base64pad'), From b2a73f7558f6e99142998718de9a4b23bb304a83 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 24 Aug 2020 14:42:05 +0100 Subject: [PATCH 37/40] chore: update ipfs-utils version --- packages/interface-ipfs-core/package.json | 2 +- packages/ipfs-core-utils/package.json | 2 +- packages/ipfs-http-client/package.json | 2 +- packages/ipfs/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/interface-ipfs-core/package.json b/packages/interface-ipfs-core/package.json index ef6874ab80..2aeda8c270 100644 --- a/packages/interface-ipfs-core/package.json +++ b/packages/interface-ipfs-core/package.json @@ -40,7 +40,7 @@ "dirty-chai": "^2.0.1", "ipfs-unixfs": "^2.0.2", "ipfs-unixfs-importer": "^3.0.2", - "ipfs-utils": "ipfs/js-ipfs-utils#revert-54-xhr", + "ipfs-utils": "^3.0.0", "ipld-block": "^0.10.0", "ipld-dag-cbor": "^0.17.0", "ipld-dag-pb": "^0.20.0", diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index dc6b23b5b4..f0f2481d95 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -32,7 +32,7 @@ "browser-readablestream-to-it": "0.0.2", "cids": "^1.0.0", "err-code": "^2.0.0", - "ipfs-utils": "ipfs/js-ipfs-utils#revert-54-xhr", + "ipfs-utils": "^3.0.0", "it-all": "^1.0.1", "it-map": "^1.0.2", "it-peekable": "0.0.1", diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 839b9e1661..dfda026463 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -47,7 +47,7 @@ "debug": "^4.1.0", "form-data": "^3.0.0", "ipfs-core-utils": "^0.3.1", - "ipfs-utils": "ipfs/js-ipfs-utils#revert-54-xhr", + "ipfs-utils": "^3.0.0", "ipld-block": "^0.10.0", "ipld-dag-cbor": "^0.17.0", "ipld-dag-pb": "^0.20.0", diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 9b1a619135..cb3f413026 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -100,7 +100,7 @@ "ipfs-unixfs": "^2.0.2", "ipfs-unixfs-exporter": "^3.0.2", "ipfs-unixfs-importer": "^3.0.2", - "ipfs-utils": "ipfs/js-ipfs-utils#revert-54-xhr", + "ipfs-utils": "^3.0.0", "ipld": "^0.27.1", "ipld-bitcoin": "^0.4.0", "ipld-block": "^0.10.0", From 27d1866e08e08119513a1d0d7d74074255accacf Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 24 Aug 2020 15:33:50 +0100 Subject: [PATCH 38/40] chore: update dep versions --- examples/custom-libp2p/package.json | 2 +- packages/ipfs/package.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/custom-libp2p/package.json b/examples/custom-libp2p/package.json index 65e10ce997..85a3a4f7c5 100644 --- a/examples/custom-libp2p/package.json +++ b/examples/custom-libp2p/package.json @@ -11,7 +11,7 @@ "license": "MIT", "dependencies": { "ipfs": "^0.49.0", - "libp2p": "libp2p/js-libp2p#fix/replace-node-buffers-with-uint8arrays", + "libp2p": "libp2p/js-libp2p#0.29.x", "libp2p-bootstrap": "^0.12.0", "libp2p-kad-dht": "^0.20.0", "libp2p-mdns": "^0.15.0", diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index cb3f413026..18ef28ec5c 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -74,7 +74,7 @@ "bl": "^4.0.2", "byteman": "^1.3.5", "cbor": "^5.0.1", - "cid-tool": "multiformats/js-cid-tool#fix/replace-node-buffers-with-uint8arrays", + "cid-tool": "^1.0.0", "cids": "^1.0.0", "class-is": "^1.1.0", "dag-cbor-links": "achingbrain/dag-cbor-links#chore/update-deps", @@ -91,7 +91,7 @@ "hapi-pino": "^6.1.0", "hashlru": "^2.3.0", "interface-datastore": "^2.0.0", - "ipfs-bitswap": "ipfs/js-ipfs-bitswap#fix/replace-node-buffers-with-uint8arrays", + "ipfs-bitswap": "^3.0.0", "ipfs-block-service": "^0.18.0", "ipfs-core-utils": "^0.3.1", "ipfs-http-client": "^46.0.0", @@ -128,7 +128,7 @@ "iterable-ndjson": "^1.1.0", "jsondiffpatch": "^0.4.1", "just-safe-set": "^2.1.0", - "libp2p": "libp2p/js-libp2p#fix/replace-node-buffers-with-uint8arrays", + "libp2p": "libp2p/js-libp2p#0.29.x", "libp2p-bootstrap": "^0.12.0", "libp2p-crypto": "^0.18.0", "libp2p-delegated-content-routing": "^0.6.0", @@ -138,7 +138,7 @@ "libp2p-kad-dht": "^0.20.0", "libp2p-mdns": "^0.15.0", "libp2p-mplex": "^0.10.0", - "libp2p-noise": "npm:@achingbrain/libp2p-noise@1.1.2", + "libp2p-noise": "^2.0.0", "libp2p-record": "^0.9.0", "libp2p-secio": "^0.13.0", "libp2p-tcp": "^0.15.0", From 6efcc3a129f7ae1aeae9b074feade1b3f3390f4c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 24 Aug 2020 16:08:30 +0100 Subject: [PATCH 39/40] chore: stringify ipns record properly --- packages/ipfs/src/core/ipns/resolver.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/ipfs/src/core/ipns/resolver.js b/packages/ipfs/src/core/ipns/resolver.js index 54cfa992db..4e119b53cb 100644 --- a/packages/ipfs/src/core/ipns/resolver.js +++ b/packages/ipfs/src/core/ipns/resolver.js @@ -6,6 +6,7 @@ const errcode = require('err-code') const debug = require('debug') const log = debug('ipfs:ipns:resolver') log.error = debug('ipfs:ipns:resolver:error') +const uint8ArrayToString = require('uint8arrays/to-string') const { Errors } = require('interface-datastore') const ERR_NOT_FOUND = Errors.notFoundError().code @@ -109,7 +110,7 @@ class IpnsResolver { // IPNS entry validation await ipns.validate(pubKey, ipnsEntry) - return ipnsEntry.value.toString() + return uint8ArrayToString(ipnsEntry.value) } } From 2cff1c8fef77da81b56c4e8864564298b854a455 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 24 Aug 2020 17:34:10 +0100 Subject: [PATCH 40/40] chore: update dep version --- packages/ipfs/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 18ef28ec5c..70702092da 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -77,7 +77,7 @@ "cid-tool": "^1.0.0", "cids": "^1.0.0", "class-is": "^1.1.0", - "dag-cbor-links": "achingbrain/dag-cbor-links#chore/update-deps", + "dag-cbor-links": "^2.0.0", "datastore-core": "^2.0.0", "datastore-pubsub": "^0.4.0", "debug": "^4.1.0",