From d832277bc3f9f2c810cfb9162e032da11aa6e60f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 4 Dec 2018 17:05:12 +0000 Subject: [PATCH] feat: add streaming option to http License: MIT Signed-off-by: achingbrain --- src/core/ls-pull-stream.js | 4 + src/http/ls.js | 71 ++++++- test/ls.spec.js | 412 +++++++++++++++++++++---------------- 3 files changed, 305 insertions(+), 182 deletions(-) diff --git a/src/core/ls-pull-stream.js b/src/core/ls-pull-stream.js index 12f6ad0b53..7c94544186 100644 --- a/src/core/ls-pull-stream.js +++ b/src/core/ls-pull-stream.js @@ -30,6 +30,10 @@ module.exports = (context) => { path = FILE_SEPARATOR } + if (path === undefined) { + path = FILE_SEPARATOR + } + options = Object.assign({}, defaultOptions, options) options.long = options.l || options.long diff --git a/src/http/ls.js b/src/http/ls.js index 54a5a25940..8886e3a53e 100644 --- a/src/http/ls.js +++ b/src/http/ls.js @@ -1,6 +1,18 @@ 'use strict' const Joi = require('joi') +const { + PassThrough +} = require('stream') + +const mapEntry = (entry) => { + return { + Name: entry.name, + Type: entry.type, + Size: entry.size, + Hash: entry.hash + } +} const mfsLs = (api) => { api.route({ @@ -14,21 +26,59 @@ const mfsLs = (api) => { const { arg, long, - cidBase + cidBase, + stream } = request.query + if (stream) { + const readableStream = ipfs.files.lsReadableStream(arg, { + long, + cidBase + }) + + if (!readableStream._read) { + // make the stream look like a Streams2 to appease Hapi + readableStream._read = () => {} + readableStream._readableState = {} + } + + let passThrough + + readableStream.on('data', (entry) => { + if (!passThrough) { + passThrough = new PassThrough() + + reply(passThrough) + .header('X-Stream-Output', '1') + } + + passThrough.write(JSON.stringify(mapEntry(entry)) + '\n') + }) + + readableStream.once('end', (entry) => { + if (passThrough) { + passThrough.end(entry ? JSON.stringify(mapEntry(entry)) + '\n' : undefined) + } + }) + + readableStream.once('error', (error) => { + reply({ + Message: error.message, + Code: error.code || 0, + Type: 'error' + }).code(500).takeover() + }) + + return + } + return ipfs.files.ls(arg, { long, cidBase }) .then(files => { reply({ - Entries: files.map(file => ({ - Name: file.name, - Type: file.type, - Size: file.size, - Hash: file.hash - })) + Entries: files.map(mapEntry) }) }) .catch(error => { @@ -47,12 +97,17 @@ const mfsLs = (api) => { query: Joi.object().keys({ arg: Joi.string().default('/'), long: Joi.boolean().default(false), - cidBase: Joi.string().default('base58btc') + cidBase: Joi.string().default('base58btc'), + stream: Joi.boolean().default(false) }) .rename('l', 'long', { override: true, ignoreUndefined: true }) + .rename('s', 'stream', { + override: true, + ignoreUndefined: true + }) } } }) diff --git a/test/ls.spec.js b/test/ls.spec.js index 8be0b38694..c99ea495ad 100644 --- a/test/ls.spec.js +++ b/test/ls.spec.js @@ -4,6 +4,8 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect +const pull = require('pull-stream/pull') +const collect = require('pull-stream/sinks/collect') const { FILE_TYPES } = require('../src') @@ -23,194 +25,256 @@ describe('ls', function () { }) }) - it('lists the root directory by default', () => { - const fileName = `small-file-${Math.random()}.txt` - const content = Buffer.from('Hello world') + const methods = [{ + name: 'ls', + ls: function () { + return mfs.ls.apply(mfs, arguments) + }, + collect: (entries) => entries + }, { + name: 'lsPullStream', + ls: function () { + return Promise.resolve(mfs.lsPullStream.apply(mfs, arguments)) + }, + collect: (stream) => { + return new Promise((resolve, reject) => { + pull( + stream, + collect((error, entries) => { + if (error) { + return reject(error) + } - return mfs.write(`/${fileName}`, content, { - create: true - }) - .then(() => mfs.ls()) - .then(files => { - expect(files.length).to.equal(1) - expect(files[0].name).to.equal(fileName) - expect(files[0].type).to.equal(FILE_TYPES.file) - expect(files[0].size).to.equal(0) - expect(files[0].hash).to.equal('') + resolve(entries) + }) + ) }) - }) + } + }, { + name: 'lsReadableStream', + ls: function () { + return Promise.resolve(mfs.lsReadableStream.apply(mfs, arguments)) + }, + collect: (stream) => { + return new Promise((resolve, reject) => { + let entries = [] - it('refuses to lists files with an empty path', () => { - return mfs.ls('') - .then(() => { - throw new Error('No error was thrown for an empty path') - }) - .catch(error => { - expect(error.message).to.contain('paths must not be empty') - }) - }) + stream.on('data', (entry) => { + entries.push(entry) + }) - it('refuses to lists files with an invalid path', () => { - return mfs.ls('not-valid') - .then(() => { - throw new Error('No error was thrown for an empty path') - }) - .catch(error => { - expect(error.message).to.contain('paths must start with a leading /') - }) - }) + stream.on('end', (entry) => { + if (entry) { + entries.push(entry) + } - it('lists files in a directory', () => { - const dirName = `dir-${Math.random()}` - const fileName = `small-file-${Math.random()}.txt` - const content = Buffer.from('Hello world') + resolve(entries) + }) - return mfs.write(`/${dirName}/${fileName}`, content, { - create: true, - parents: true - }) - .then(() => mfs.ls(`/${dirName}`, {})) - .then(files => { - expect(files.length).to.equal(1) - expect(files[0].name).to.equal(fileName) - expect(files[0].type).to.equal(FILE_TYPES.file) - expect(files[0].size).to.equal(0) - expect(files[0].hash).to.equal('') + stream.on('error', (error) => { + reject(error) + }) }) - }) + } + }] - it('lists files in a directory with meta data', () => { - const dirName = `dir-${Math.random()}` - const fileName = `small-file-${Math.random()}.txt` - const content = Buffer.from('Hello world') - - return mfs.write(`/${dirName}/${fileName}`, content, { - create: true, - parents: true - }) - .then(() => mfs.ls(`/${dirName}`, { - long: true - })) - .then(files => { - expect(files.length).to.equal(1) - expect(files[0].name).to.equal(fileName) - expect(files[0].type).to.equal(FILE_TYPES.file) - expect(files[0].size).to.equal(content.length) + methods.forEach(method => { + describe(`ls ${method.name}`, function () { + it('lists the root directory by default', () => { + const fileName = `small-file-${Math.random()}.txt` + const content = Buffer.from('Hello world') + + return mfs.write(`/${fileName}`, content, { + create: true + }) + .then(() => method.ls()) + .then((result) => method.collect(result)) + .then(files => { + expect(files.find(file => file.name === fileName)).to.be.ok() + }) }) - }) - - it('lists a file', () => { - const fileName = `small-file-${Math.random()}.txt` - const content = Buffer.from('Hello world') - - return mfs.write(`/${fileName}`, content, { - create: true - }) - .then(() => mfs.ls(`/${fileName}`)) - .then(files => { - expect(files.length).to.equal(1) - expect(files[0].name).to.equal(fileName) - expect(files[0].type).to.equal(FILE_TYPES.file) - expect(files[0].size).to.equal(0) - expect(files[0].hash).to.equal('') + + it('refuses to lists files with an empty path', () => { + return method.ls('') + .then((result) => method.collect(result)) + .then(() => { + throw new Error('No error was thrown for an empty path') + }) + .catch(error => { + expect(error.message).to.contain('paths must not be empty') + }) }) - }) - - it('lists a file with meta data', () => { - const fileName = `small-file-${Math.random()}.txt` - const content = Buffer.from('Hello world') - - return mfs.write(`/${fileName}`, content, { - create: true - }) - .then(() => mfs.ls(`/${fileName}`, { - long: true - })) - .then(files => { - expect(files.length).to.equal(1) - expect(files[0].name).to.equal(fileName) - expect(files[0].type).to.equal(FILE_TYPES.file) - expect(files[0].size).to.equal(content.length) + + it('refuses to lists files with an invalid path', () => { + return method.ls('not-valid') + .then((result) => method.collect(result)) + .then(() => { + throw new Error('No error was thrown for an empty path') + }) + .catch(error => { + expect(error.message).to.contain('paths must start with a leading /') + }) }) - }) - - it('lists a file with a base32 hash', () => { - const fileName = `small-file-${Math.random()}.txt` - const content = Buffer.from('Hello world') - - return mfs.write(`/${fileName}`, content, { - create: true - }) - .then(() => mfs.ls(`/${fileName}`, { - long: true, - cidBase: 'base32' - })) - .then(files => { - expect(files.length).to.equal(1) - expect(files[0].name).to.equal(fileName) - expect(files[0].type).to.equal(FILE_TYPES.file) - expect(files[0].size).to.equal(content.length) - expect(files[0].hash.startsWith('b')).to.equal(true) + + it('lists files in a directory', () => { + const dirName = `dir-${Math.random()}` + const fileName = `small-file-${Math.random()}.txt` + const content = Buffer.from('Hello world') + + return mfs.write(`/${dirName}/${fileName}`, content, { + create: true, + parents: true + }) + .then(() => method.ls(`/${dirName}`, {})) + .then((result) => method.collect(result)) + .then(files => { + expect(files.length).to.equal(1) + expect(files[0].name).to.equal(fileName) + expect(files[0].type).to.equal(FILE_TYPES.file) + expect(files[0].size).to.equal(0) + expect(files[0].hash).to.equal('') + }) }) - }) - - it('fails to list non-existent file', () => { - return mfs.ls('/i-do-not-exist') - .then(() => { - throw new Error('No error was thrown for a non-existent file') + + it('lists files in a directory with meta data', () => { + const dirName = `dir-${Math.random()}` + const fileName = `small-file-${Math.random()}.txt` + const content = Buffer.from('Hello world') + + return mfs.write(`/${dirName}/${fileName}`, content, { + create: true, + parents: true + }) + .then(() => method.ls(`/${dirName}`, { + long: true + })) + .then((result) => method.collect(result)) + .then(files => { + expect(files.length).to.equal(1) + expect(files[0].name).to.equal(fileName) + expect(files[0].type).to.equal(FILE_TYPES.file) + expect(files[0].size).to.equal(content.length) + }) }) - .catch(error => { - expect(error.message).to.contain('does not exist') + + it('lists a file', () => { + const fileName = `small-file-${Math.random()}.txt` + const content = Buffer.from('Hello world') + + return mfs.write(`/${fileName}`, content, { + create: true + }) + .then(() => method.ls(`/${fileName}`)) + .then((result) => method.collect(result)) + .then(files => { + expect(files.length).to.equal(1) + expect(files[0].name).to.equal(fileName) + expect(files[0].type).to.equal(FILE_TYPES.file) + expect(files[0].size).to.equal(0) + expect(files[0].hash).to.equal('') + }) + }) + + it('lists a file with meta data', () => { + const fileName = `small-file-${Math.random()}.txt` + const content = Buffer.from('Hello world') + + return mfs.write(`/${fileName}`, content, { + create: true + }) + .then(() => method.ls(`/${fileName}`, { + long: true + })) + .then((result) => method.collect(result)) + .then(files => { + expect(files.length).to.equal(1) + expect(files[0].name).to.equal(fileName) + expect(files[0].type).to.equal(FILE_TYPES.file) + expect(files[0].size).to.equal(content.length) + }) + }) + + it('lists a file with a base32 hash', () => { + const fileName = `small-file-${Math.random()}.txt` + const content = Buffer.from('Hello world') + + return mfs.write(`/${fileName}`, content, { + create: true + }) + .then(() => method.ls(`/${fileName}`, { + long: true, + cidBase: 'base32' + })) + .then((result) => method.collect(result)) + .then(files => { + expect(files.length).to.equal(1) + expect(files[0].name).to.equal(fileName) + expect(files[0].type).to.equal(FILE_TYPES.file) + expect(files[0].size).to.equal(content.length) + expect(files[0].hash.startsWith('b')).to.equal(true) + }) + }) + + it('fails to list non-existent file', () => { + return method.ls('/i-do-not-exist') + .then((result) => method.collect(result)) + .then(() => { + throw new Error('No error was thrown for a non-existent file') + }) + .catch(error => { + expect(error.message).to.contain('does not exist') + }) + }) + + it('lists a sharded directory contents', async () => { + const shardSplitThreshold = 10 + const fileCount = 11 + const dirPath = await createShardedDirectory(mfs, shardSplitThreshold, fileCount) + + const files = await method.collect(await method.ls(dirPath, { + long: true + })) + + expect(files.length).to.equal(fileCount) + + files.forEach(file => { + // should be a file + expect(file.type).to.equal(0) + }) + }) + + it('lists a file inside a sharded directory directly', async () => { + const dirPath = await createShardedDirectory(mfs) + + const files = await method.collect(await method.ls(dirPath, { + long: true + })) + + const filePath = `${dirPath}/${files[0].name}` + + // should be able to ls new file directly + expect(await method.collect(await method.ls(filePath, { + long: true + }))).to.not.be.empty() + }) + + it('lists the contents of a directory inside a sharded directory', async () => { + const shardedDirPath = await createShardedDirectory(mfs) + const dirPath = `${shardedDirPath}/subdir-${Math.random()}` + const fileName = `small-file-${Math.random()}.txt` + + await mfs.mkdir(`${dirPath}`) + await mfs.write(`${dirPath}/${fileName}`, Buffer.from([0, 1, 2, 3]), { + create: true + }) + + const files = await method.collect(await method.ls(dirPath, { + long: true + })) + + expect(files.length).to.equal(1) + expect(files.filter(file => file.name === fileName)).to.be.ok() }) - }) - - it('lists a sharded directory contents', async () => { - const shardSplitThreshold = 10 - const fileCount = 11 - const dirPath = await createShardedDirectory(mfs, shardSplitThreshold, fileCount) - - const files = await mfs.ls(dirPath, { - long: true - }) - - expect(files.length).to.equal(fileCount) - - files.forEach(file => { - // should be a file - expect(file.type).to.equal(0) - }) - }) - - it('lists a file inside a sharded directory directly', async () => { - const dirPath = await createShardedDirectory(mfs) - - const files = await mfs.ls(dirPath, { - long: true - }) - - const filePath = `${dirPath}/${files[0].name}` - - // should be able to ls new file directly - expect(await mfs.ls(filePath, { - long: true - })).to.not.be.empty() - }) - - it('lists the contents of a directory inside a sharded directory', async () => { - const shardedDirPath = await createShardedDirectory(mfs) - const dirPath = `${shardedDirPath}/subdir-${Math.random()}` - const fileName = `small-file-${Math.random()}.txt` - - await mfs.mkdir(`${dirPath}`) - await mfs.write(`${dirPath}/${fileName}`, Buffer.from([0, 1, 2, 3]), { - create: true - }) - - const files = await mfs.ls(dirPath, { - long: true }) - - expect(files.length).to.equal(1) - expect(files.filter(file => file.name === fileName)).to.be.ok() }) })