From 524a110e6d63affad06c84c780a239ab24857eae Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 5 Sep 2019 12:40:28 +0200 Subject: [PATCH] refactor: tests of addFromURL in browsers (#514) > Required by https://github.com/ipfs/js-ipfs/pull/2304 Old version of `addFromURL` tests was Node-centric and did not work in web browser contexts. This PR: - moves the HTTP server to a separate file to enable spawning it from aegir hook (see .aegir.js/hooks/browser/pre|post in js-ipfs from https://github.com/ipfs/js-ipfs/pull/2304) - simplifies the way we use HTTP Echo Server by moving it to aegir hooks and removing HTTPS version - https://github.com/ipfs/interface-js-ipfs-core/pull/514#issuecomment-526225635 - removes overhead of starting and stopping HTTP servers multiple times - shortens/simplifies test code a bit License: MIT Signed-off-by: Marcin Rataj --- src/files-regular/add-from-url.js | 285 ++++++------------------------ src/files-regular/utils.js | 4 - src/utils/echo-http-server.js | 62 +++++++ 3 files changed, 120 insertions(+), 231 deletions(-) create mode 100644 src/utils/echo-http-server.js diff --git a/src/files-regular/add-from-url.js b/src/files-regular/add-from-url.js index 47b5b62f57..49bb5119c8 100644 --- a/src/files-regular/add-from-url.js +++ b/src/files-regular/add-from-url.js @@ -1,13 +1,9 @@ /* eslint-env mocha */ 'use strict' -const { fixtures } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') -const http = require('http') -const https = require('https') -const each = require('async/each') -const waterfall = require('async/waterfall') const parallel = require('async/parallel') +const { echoUrl, redirectUrl } = require('../utils/echo-http-server') module.exports = (createCommon, options) => { const describe = getDescribe(options) @@ -23,10 +19,8 @@ module.exports = (createCommon, options) => { // CI takes longer to instantiate the daemon, so we need to increase the // timeout for the before step this.timeout(60 * 1000) - common.setup((err, factory) => { expect(err).to.not.exist() - factory.spawnNode((err, node) => { expect(err).to.not.exist() ipfs = node @@ -37,260 +31,97 @@ module.exports = (createCommon, options) => { after((done) => common.teardown(done)) - let testServers = [] - - const sslOpts = fixtures.sslOpts - - const startTestServer = (handler, opts, cb) => { - if (typeof opts === 'function') { - cb = opts - opts = {} - } - - const server = opts.secure - ? https.createServer(sslOpts, handler) - : http.createServer(handler) - - server.listen((err) => { - if (err) return cb(err) - testServers.push(server) - cb(null, server) - }) - } - - beforeEach(() => { - // Instructs node to not reject our snake oil SSL certificate when it - // can't verify the certificate authority - process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0 - }) - - afterEach((done) => { - // Reinstate unauthorised SSL cert rejection - process.env.NODE_TLS_REJECT_UNAUTHORIZED = 1 - - each(testServers, (server, cb) => server.close(cb), (err) => { - testServers = [] - done(err) - }) - }) - it('should add from a HTTP URL', (done) => { - const data = Buffer.from(`TEST${Date.now()}`) - + const text = `TEST${Date.now()}` + const url = echoUrl(text) parallel({ - server: (cb) => { - const handler = (req, res) => { - res.write(data) - res.end() - } - startTestServer(handler, cb) - }, - expectedResult: (cb) => ipfs.add(data, cb) - }, (err, taskResult) => { + result: (cb) => ipfs.addFromURL(url, cb), + expectedResult: (cb) => ipfs.add(Buffer.from(text), cb) + }, (err, { result, expectedResult }) => { expect(err).to.not.exist() - const { server, expectedResult } = taskResult - - const url = `http://127.0.0.1:${server.address().port}/` - ipfs.addFromURL(url, (err, result) => { - expect(err).to.not.exist() - expect(result).to.deep.equal(expectedResult) - done() - }) - }) - }) - - it('should add from a HTTPS URL', (done) => { - const data = Buffer.from(`TEST${Date.now()}`) - - parallel({ - server: (cb) => { - const handler = (req, res) => { - res.write(data) - res.end() - } - startTestServer(handler, { secure: true }, cb) - }, - expectedResult: (cb) => ipfs.add(data, cb) - }, (err, taskResult) => { - expect(err).to.not.exist() - const { server, expectedResult } = taskResult - - const url = `https://127.0.0.1:${server.address().port}/` - ipfs.addFromURL(url, (err, result) => { - expect(err).to.not.exist() - expect(result).to.deep.equal(expectedResult) - done() - }) + expect(result.err).to.not.exist() + expect(expectedResult.err).to.not.exist() + expect(result[0].hash).to.equal(expectedResult[0].hash) + expect(result[0].size).to.equal(expectedResult[0].size) + expect(result[0].path).to.equal(text) + done() }) }) it('should add from a HTTP URL with redirection', (done) => { - const data = Buffer.from(`TEST${Date.now()}`) - - waterfall([ - (cb) => { - const handler = (req, res) => { - res.write(data) - res.end() - } - startTestServer(handler, cb) - }, - (serverA, cb) => { - const url = `http://127.0.0.1:${serverA.address().port}` - const handler = (req, res) => { - res.statusCode = 302 - res.setHeader('Location', url) - res.end() - } - startTestServer(handler, (err, serverB) => { - if (err) return cb(err) - cb(null, { a: serverA, b: serverB }) - }) - } - ], (err, servers) => { - expect(err).to.not.exist() - - ipfs.add(data, (err, res) => { - expect(err).to.not.exist() - - const expectedHash = res[0].hash - const url = `http://127.0.0.1:${servers.b.address().port}` + const text = `TEST${Date.now()}` + const url = echoUrl(text) + '?foo=bar#buzz' - ipfs.addFromURL(url, (err, result) => { - expect(err).to.not.exist() - expect(result[0].hash).to.equal(expectedHash) - done() - }) - }) - }) - }) - - it('should add from a HTTPS URL with redirection', (done) => { - const data = Buffer.from(`TEST${Date.now()}`) - - waterfall([ - (cb) => { - const handler = (req, res) => { - res.write(data) - res.end() - } - startTestServer(handler, { secure: true }, cb) - }, - (serverA, cb) => { - const url = `https://127.0.0.1:${serverA.address().port}` - const handler = (req, res) => { - res.statusCode = 302 - res.setHeader('Location', url) - res.end() - } - startTestServer(handler, { secure: true }, (err, serverB) => { - if (err) return cb(err) - cb(null, { a: serverA, b: serverB }) - }) - } - ], (err, servers) => { + parallel({ + result: (cb) => ipfs.addFromURL(redirectUrl(url), cb), + expectedResult: (cb) => ipfs.add(Buffer.from(text), cb) + }, (err, { result, expectedResult }) => { expect(err).to.not.exist() - - ipfs.add(data, (err, res) => { - expect(err).to.not.exist() - - const expectedHash = res[0].hash - const url = `https://127.0.0.1:${servers.b.address().port}` - - ipfs.addFromURL(url, (err, result) => { - expect(err).to.not.exist() - expect(result[0].hash).to.equal(expectedHash) - done() - }) - }) + expect(result.err).to.not.exist() + expect(expectedResult.err).to.not.exist() + expect(result[0].hash).to.equal(expectedResult[0].hash) + expect(result[0].size).to.equal(expectedResult[0].size) + expect(result[0].path).to.equal(text) + done() }) }) it('should add from a URL with only-hash=true', (done) => { - const handler = (req, res) => { - res.write(`TEST${Date.now()}`) - res.end() - } - - startTestServer(handler, (err, server) => { + const text = `TEST${Date.now()}` + const url = echoUrl(text) + ipfs.addFromURL(url, { onlyHash: true }, (err, res) => { expect(err).to.not.exist() - const url = `http://127.0.0.1:${server.address().port}/` - - ipfs.addFromURL(url, { onlyHash: true }, (err, res) => { - expect(err).to.not.exist() - - // A successful object.get for this size data took my laptop ~14ms - let didTimeout = false - const timeoutId = setTimeout(() => { - didTimeout = true - done() - }, 500) + // A successful object.get for this size data took my laptop ~14ms + let didTimeout = false + const timeoutId = setTimeout(() => { + didTimeout = true + done() + }, 500) - ipfs.object.get(res[0].hash, () => { - clearTimeout(timeoutId) - if (didTimeout) return - expect(new Error('did not timeout')).to.not.exist() - }) + ipfs.object.get(res[0].hash, () => { + clearTimeout(timeoutId) + if (didTimeout) return + expect(new Error('did not timeout')).to.not.exist() }) }) }) it('should add from a URL with wrap-with-directory=true', (done) => { - const filename = `TEST${Date.now()}.txt` - const data = Buffer.from(`TEST${Date.now()}`) - + const filename = `TEST${Date.now()}.txt` // also acts as data + const url = echoUrl(filename) + '?foo=bar#buzz' + const addOpts = { wrapWithDirectory: true } parallel({ - server: (cb) => startTestServer((req, res) => { - res.write(data) - res.end() - }, cb), - expectedResult: (cb) => { - ipfs.add([{ path: filename, content: data }], { wrapWithDirectory: true }, cb) - } - }, (err, taskResult) => { + result: (cb) => ipfs.addFromURL(url, addOpts, cb), + expectedResult: (cb) => ipfs.add([{ path: filename, content: Buffer.from(filename) }], addOpts, cb) + }, (err, { result, expectedResult }) => { expect(err).to.not.exist() - - const { server, expectedResult } = taskResult - const url = `http://127.0.0.1:${server.address().port}/${filename}?foo=bar#buzz` - - ipfs.addFromURL(url, { wrapWithDirectory: true }, (err, result) => { - expect(err).to.not.exist() - expect(result).to.deep.equal(expectedResult) - done() - }) + expect(result.err).to.not.exist() + expect(expectedResult.err).to.not.exist() + expect(result).to.deep.equal(expectedResult) + done() }) }) it('should add from a URL with wrap-with-directory=true and URL-escaped file name', (done) => { - const filename = '320px-Domažlice,_Jiráskova_43_(9102).jpg' - const data = Buffer.from(`TEST${Date.now()}`) - + const filename = `320px-Domažlice,_Jiráskova_43_(${Date.now()}).jpg` // also acts as data + const url = echoUrl(filename) + '?foo=bar#buzz' + const addOpts = { wrapWithDirectory: true } parallel({ - server: (cb) => startTestServer((req, res) => { - res.write(data) - res.end() - }, cb), - expectedResult: (cb) => { - ipfs.add([{ path: filename, content: data }], { wrapWithDirectory: true }, cb) - } - }, (err, taskResult) => { + result: (cb) => ipfs.addFromURL(url, addOpts, cb), + expectedResult: (cb) => ipfs.add([{ path: filename, content: Buffer.from(filename) }], addOpts, cb) + }, (err, { result, expectedResult }) => { expect(err).to.not.exist() - - const { server, expectedResult } = taskResult - const url = `http://127.0.0.1:${server.address().port}/${encodeURIComponent(filename)}?foo=bar#buzz` - - ipfs.addFromURL(url, { wrapWithDirectory: true }, (err, result) => { - expect(err).to.not.exist() - expect(result).to.deep.equal(expectedResult) - done() - }) + expect(result.err).to.not.exist() + expect(expectedResult.err).to.not.exist() + expect(result).to.deep.equal(expectedResult) + done() }) }) it('should not add from an invalid url', (done) => { ipfs.addFromURL('http://invalid', (err, result) => { - expect(err.code).to.equal('ENOTFOUND') + expect(err).to.exist() expect(result).to.not.exist() done() }) diff --git a/src/files-regular/utils.js b/src/files-regular/utils.js index 8f6bf04d93..8ebfb229eb 100644 --- a/src/files-regular/utils.js +++ b/src/files-regular/utils.js @@ -21,9 +21,5 @@ exports.fixtures = Object.freeze({ bigFile: Object.freeze({ cid: 'Qme79tX2bViL26vNjPsF3DP1R9rMKMvnPYJiKTTKPrXJjq', data: loadFixture('test/fixtures/15mb.random', 'interface-ipfs-core') - }), - sslOpts: Object.freeze({ - key: loadFixture('test/fixtures/ssl/privkey.pem', 'interface-ipfs-core'), - cert: loadFixture('test/fixtures/ssl/cert.pem', 'interface-ipfs-core') }) }) diff --git a/src/utils/echo-http-server.js b/src/utils/echo-http-server.js new file mode 100644 index 0000000000..0554e18bde --- /dev/null +++ b/src/utils/echo-http-server.js @@ -0,0 +1,62 @@ +/* eslint-env browser */ +'use strict' + +const http = require('http') +const URL = require('url').URL || self.URL + +const defaultPort = 11080 + +// Create a mock of remote HTTP server that can return arbitrary text in response +// or redirect to other URL. Used in tests of ipfs.addFromURL etc +// It needs to be available to tests run in browsers: +// start it from Node via .aegir.js/hooks/browser/pre|post (example in js-ipfs) +module.exports.createServer = () => { + const handler = (req, res) => { + // Relaxed CORS to enable use in tests in web browser with fetch + res.setHeader('Access-Control-Allow-Origin', '*') + res.setHeader('Access-Control-Request-Method', '*') + res.setHeader('Access-Control-Allow-Methods', 'OPTIONS, GET, DELETE') + res.setHeader('Access-Control-Allow-Headers', '*') + if (req.method === 'OPTIONS') { + res.writeHead(200) + res.end() + return + } + // get the path without query or hash + const { pathname } = new URL(`https://127.0.0.1${req.url}`) + if (pathname.startsWith('/echo/')) { + // Respond with text passed in URL after /echo/ + const [, text] = pathname.split('/echo/') + res.setHeader('Content-Type', 'text/plain') + res.write(decodeURIComponent(text)) + } else if (req.url.startsWith('/302/')) { + // Return a redirect to a passed URL + const [, location] = pathname.split('/302/') + const url = decodeURI(location) + res.statusCode = 302 + res.setHeader('Location', url) + } else { + res.statusCode = 500 + } + res.end() + } + + const server = http.createServer(handler) + + server.start = (opts, cb) => { + if (typeof opts === 'function') { + cb = opts + opts = {} + } + return server.listen(Object.assign({ port: defaultPort, host: '127.0.0.1' }, opts), cb) + } + + server.stop = (cb) => server.close(cb) + + return server +} + +module.exports.defaultPort = defaultPort +module.exports.url = `http://127.0.0.1:${module.exports.defaultPort}` +module.exports.echoUrl = (text) => `${module.exports.url}/echo/${encodeURIComponent(text)}` +module.exports.redirectUrl = (url) => `${module.exports.url}/302/${encodeURI(url)}`