From 7b5c85d8f071ce8f727d11498ebddea9e544b999 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 27 Nov 2019 18:38:28 +0000 Subject: [PATCH 1/8] feat: support UnixFSv1.5 metadata --- package.json | 4 +- src/parser.js | 119 ++++++++++++++++++++++++++++---------------- test/parser.spec.js | 68 +++++++++++++++++-------- 3 files changed, 126 insertions(+), 65 deletions(-) diff --git a/package.json b/package.json index 8ca9d00..fb1d7b8 100644 --- a/package.json +++ b/package.json @@ -27,12 +27,12 @@ }, "dependencies": { "@hapi/content": "^4.1.0", - "it-multipart": "~0.0.2" + "it-multipart": "^1.0.1" }, "devDependencies": { "aegir": "^20.0.0", "chai": "^4.2.0", - "ipfs-http-client": "^35.1.0", + "ipfs-http-client": "ipfs/js-ipfs-htt-client#support-unixfs-metadata", "request": "^2.88.0" }, "engines": { diff --git a/src/parser.js b/src/parser.js index 86ea02b..cd078b4 100644 --- a/src/parser.js +++ b/src/parser.js @@ -12,22 +12,16 @@ const isDirectory = (mediatype) => mediatype === multipartFormdataType || mediat const parseDisposition = (disposition) => { const details = {} details.type = disposition.split(';')[0] - if (details.type === 'file' || details.type === 'form-data') { - const namePattern = / filename="(.[^"]+)"/ - const matches = disposition.match(namePattern) - details.name = matches ? matches[1] : '' - } - - return details -} -const parseHeader = (header) => { - const type = Content.type(header['content-type']) - const disposition = parseDisposition(header['content-disposition']) + if (details.type === 'file' || details.type === 'form-data') { + const filenamePattern = / filename="(.[^"]+)"/ + const filenameMatches = disposition.match(filenamePattern) + details.filename = filenameMatches ? filenameMatches[1] : '' - const details = type - details.name = decodeURIComponent(disposition.name) - details.type = disposition.type + const namePattern = / name="(.[^"]+)"/ + const nameMatches = disposition.match(namePattern) + details.name = nameMatches ? nameMatches[1] : '' + } return details } @@ -50,49 +44,90 @@ const ignore = async (stream) => { } } -async function * parser (stream, options) { - for await (const part of multipart(stream, options.boundary)) { - const partHeader = parseHeader(part.headers) +async function * parseEntry (stream, options) { + let entry = {} - if (isDirectory(partHeader.mime)) { - yield { - type: 'directory', - name: partHeader.name + for await (const part of stream) { + let type + + if (part.headers['content-type']) { + type = Content.type(part.headers['content-type']) + + if (type.boundary) { + // recursively parse nested multiparts + yield * parser(part.body, { + ...options, + boundary: type.boundary + }) + + continue } + } + + if (!part.headers['content-disposition']) { + throw new Error('No content disposition in multipart part') + } - await ignore(part.body) + const disposition = parseDisposition(part.headers['content-disposition']) - continue + if (disposition.name.includes('mtime')) { + entry.mtime = parseInt((await collect(part.body)).toString('utf8'), 10) } - if (partHeader.mime === applicationSymlink) { - const target = await collect(part.body) + if (disposition.name.includes('mode')) { + entry.mode = parseInt((await collect(part.body)).toString('utf8'), 10) + } - yield { - type: 'symlink', - name: partHeader.name, - target: target.toString('utf8') + if (type) { + if (isDirectory(type.mime)) { + entry.type = 'directory' + } else if (type.mime === applicationSymlink) { + entry.type = 'symlink' + } else { + entry.type = 'file' } - continue + entry.name = decodeURIComponent(disposition.filename) + entry.body = part.body + + yield entry + + entry = {} } + } +} - if (partHeader.boundary) { - // recursively parse nested multiparts - for await (const entry of parser(part, { - ...options, - boundary: partHeader.boundary - })) { - yield entry +async function * parser (stream, options) { + for await (const entry of parseEntry(multipart(stream, options.boundary), options)) { + if (entry.type === 'directory') { + yield { + type: 'directory', + name: entry.name, + mtime: entry.mtime, + mode: entry.mode } - continue + await ignore(entry.body) + } + + if (entry.type === 'symlink') { + yield { + type: 'symlink', + name: entry.name, + target: (await collect(entry.body)).toString('utf8'), + mtime: entry.mtime, + mode: entry.mode + } } - yield { - type: 'file', - name: partHeader.name, - content: part.body + if (entry.type === 'file') { + yield { + type: 'file', + name: entry.name, + content: entry.body, + mtime: entry.mtime, + mode: entry.mode + } } } } diff --git a/test/parser.spec.js b/test/parser.spec.js index 2214306..565254e 100644 --- a/test/parser.spec.js +++ b/test/parser.spec.js @@ -14,7 +14,7 @@ const os = require('os') const isWindows = os.platform() === 'win32' -const readDir = (path, prefix, output = []) => { +const readDir = (path, prefix, includeMetadata, output = []) => { const entries = fs.readdirSync(path) entries.forEach(entry => { @@ -23,21 +23,25 @@ const readDir = (path, prefix, output = []) => { const type = fs.statSync(entryPath) if (type.isDirectory()) { - readDir(entryPath, `${prefix}/${entry}`, output) + readDir(entryPath, `${prefix}/${entry}`, includeMetadata, output) + + output.push({ + path: `${prefix}/${entry}`, + mtime: includeMetadata ? parseInt(type.mtimeMs / 1000) : undefined, + mode: includeMetadata ? type.mode : undefined + }) } if (type.isFile()) { output.push({ path: `${prefix}/${entry}`, - content: fs.createReadStream(entryPath) + content: fs.createReadStream(entryPath), + mtime: includeMetadata ? parseInt(type.mtimeMs / 1000) : undefined, + mode: includeMetadata ? type.mode : undefined }) } }) - output.push({ - path: prefix - }) - return output } @@ -75,6 +79,8 @@ describe('parser', () => { describe('single file', () => { const filePath = path.resolve(__dirname, 'fixtures/config') const fileContent = fs.readFileSync(filePath, 'utf8') + const fileMtime = parseInt(Date.now() / 1000) + const fileMode = parseInt('0777', 8) before(() => { handler = async (req) => { @@ -84,7 +90,7 @@ describe('parser', () => { for await (const entry of parser(req)) { if (entry.type === 'file') { - const file = { name: entry.name, content: '' } + const file = { ...entry, content: '' } for await (const data of entry.content) { file.content += data.toString() @@ -95,17 +101,18 @@ describe('parser', () => { } expect(files.length).to.equal(1) - expect(files[0].name).to.equal('config') - expect(files[0].content).to.equal(fileContent) + expect(JSON.parse(files[0].content)).to.deep.equal(JSON.parse(fileContent)) } }) it('parses ctl.config.replace correctly', async () => { - await ctl.config.replace(filePath) + await ctl.config.replace(JSON.parse(fileContent)) }) it('parses regular multipart requests correctly', (done) => { const formData = { + mtime: fileMtime, + mode: fileMode, file: fs.createReadStream(filePath) } @@ -123,15 +130,15 @@ describe('parser', () => { expect(req.headers['content-type']).to.be.a('string') for await (const entry of parser(req)) { - if (entry.type === 'file') { - const file = { name: entry.name, content: '' } + const file = { ...entry, content: '' } + if (entry.content) { for await (const data of entry.content) { file.content += data.toString() } - - files.push(file) } + + files.push(file) } } }) @@ -149,12 +156,31 @@ describe('parser', () => { return } - expect(files.length).to.equal(5) - expect(files[0].name).to.equal('fixtures/config') - expect(files[1].name).to.equal('fixtures/folderlink/deepfile') - expect(files[2].name).to.equal('fixtures/link') - expect(files[3].name).to.equal('fixtures/otherfile') - expect(files[4].name).to.equal('fixtures/subfolder/deepfile') + expect(files).to.have.lengthOf(contents.length) + + for (let i = 0; i < contents.length; i++) { + expect(files[i].name).to.equal(contents[i].path) + expect(files[i].mode).to.be.undefined + expect(files[i].mtime).to.be.undefined + } + }) + + it('parses ctl.add with metadata correctly', async () => { + const contents = readDir(dirPath, 'fixtures', true) + + await ctl.add(contents, { recursive: true, followSymlinks: false }) + + if (isWindows) { + return + } + + expect(files).to.have.lengthOf(contents.length) + + for (let i = 0; i < contents.length; i++) { + expect(files[i].name).to.equal(contents[i].path) + expect(files[i].mode).to.equal(contents[i].mode) + expect(files[i].mtime).to.equal(contents[i].mtime) + } }) }) From 93ec9c0abaf36e7b244046073fd397ce3a4709af Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 27 Nov 2019 18:43:59 +0000 Subject: [PATCH 2/8] chore: use correct repo name --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fb1d7b8..cedb820 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "devDependencies": { "aegir": "^20.0.0", "chai": "^4.2.0", - "ipfs-http-client": "ipfs/js-ipfs-htt-client#support-unixfs-metadata", + "ipfs-http-client": "ipfs/js-ipfs-http-client#support-unixfs-metadata", "request": "^2.88.0" }, "engines": { From 41557966c0f4f8cf2fef6aa5e0942eb54da43a4d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 4 Dec 2019 14:43:18 +0000 Subject: [PATCH 3/8] refactor: pass mode and mtime in headers --- src/parser.js | 58 +++++++++++++++++++++------------------------ test/parser.spec.js | 19 +++++++++++++-- 2 files changed, 44 insertions(+), 33 deletions(-) diff --git a/src/parser.js b/src/parser.js index cd078b4..7b23d4b 100644 --- a/src/parser.js +++ b/src/parser.js @@ -45,55 +45,51 @@ const ignore = async (stream) => { } async function * parseEntry (stream, options) { - let entry = {} - for await (const part of stream) { - let type + if (!part.headers['content-type']) { + throw new Error('No content-type in multipart part') + } - if (part.headers['content-type']) { - type = Content.type(part.headers['content-type']) + const type = Content.type(part.headers['content-type']) - if (type.boundary) { - // recursively parse nested multiparts - yield * parser(part.body, { - ...options, - boundary: type.boundary - }) + if (type.boundary) { + // recursively parse nested multiparts + yield * parser(part.body, { + ...options, + boundary: type.boundary + }) - continue - } + continue } if (!part.headers['content-disposition']) { throw new Error('No content disposition in multipart part') } - const disposition = parseDisposition(part.headers['content-disposition']) + const entry = {} - if (disposition.name.includes('mtime')) { - entry.mtime = parseInt((await collect(part.body)).toString('utf8'), 10) + if (part.headers.mtime) { + entry.mtime = parseInt(part.headers.mtime, 10) } - if (disposition.name.includes('mode')) { - entry.mode = parseInt((await collect(part.body)).toString('utf8'), 10) + if (part.headers.mode) { + entry.mode = parseInt(part.headers.mode, 8) } - if (type) { - if (isDirectory(type.mime)) { - entry.type = 'directory' - } else if (type.mime === applicationSymlink) { - entry.type = 'symlink' - } else { - entry.type = 'file' - } + if (isDirectory(type.mime)) { + entry.type = 'directory' + } else if (type.mime === applicationSymlink) { + entry.type = 'symlink' + } else { + entry.type = 'file' + } - entry.name = decodeURIComponent(disposition.filename) - entry.body = part.body + const disposition = parseDisposition(part.headers['content-disposition']) - yield entry + entry.name = decodeURIComponent(disposition.filename) + entry.body = part.body - entry = {} - } + yield entry } } diff --git a/test/parser.spec.js b/test/parser.spec.js index 565254e..2889864 100644 --- a/test/parser.spec.js +++ b/test/parser.spec.js @@ -111,13 +111,28 @@ describe('parser', () => { it('parses regular multipart requests correctly', (done) => { const formData = { - mtime: fileMtime, - mode: fileMode, file: fs.createReadStream(filePath) } request.post({ url: `http://localhost:${PORT}`, formData: formData }, (err) => done(err)) }) + + it('parses multipart requests with metatdata correctly', (done) => { + const r = request.post({ url: `http://localhost:${PORT}` }, (err) => done(err)) + + // request uses an old version of form-data so this is clunky + const CRLF = '\r\n' + const form = r.form() + form.append('file', fileContent, { + header: [ + `--${form.getBoundary()}`, + 'content-type: application/octet-stream', + 'content-disposition: form-data; filename="file.txt"; name="file"', + `mtime: ${fileMtime}`, + `mode: ${fileMode}` + ].join(CRLF) + CRLF + }) + }) }) describe('directory', () => { From d877ff6eb522a1e7ee7277afdecb5ba0b12958e5 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 4 Dec 2019 14:47:49 +0000 Subject: [PATCH 4/8] refactor: refactor clunky test --- test/parser.spec.js | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/test/parser.spec.js b/test/parser.spec.js index 2889864..1e4fed8 100644 --- a/test/parser.spec.js +++ b/test/parser.spec.js @@ -118,20 +118,19 @@ describe('parser', () => { }) it('parses multipart requests with metatdata correctly', (done) => { - const r = request.post({ url: `http://localhost:${PORT}` }, (err) => done(err)) - - // request uses an old version of form-data so this is clunky - const CRLF = '\r\n' - const form = r.form() - form.append('file', fileContent, { - header: [ - `--${form.getBoundary()}`, - 'content-type: application/octet-stream', - 'content-disposition: form-data; filename="file.txt"; name="file"', - `mtime: ${fileMtime}`, - `mode: ${fileMode}` - ].join(CRLF) + CRLF - }) + const formData = { + file: { + value: fileContent, + options: { + header: { + mtime: fileMtime, + mode: fileMode + } + } + } + } + + request.post({ url: `http://localhost:${PORT}`, formData }, (err) => done(err)) }) }) From 46f3a63e4d2604aa357501b37dfb80cf8a10eddc Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 4 Dec 2019 14:48:19 +0000 Subject: [PATCH 5/8] chore: fix linting --- test/parser.spec.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/parser.spec.js b/test/parser.spec.js index 1e4fed8..1a79072 100644 --- a/test/parser.spec.js +++ b/test/parser.spec.js @@ -122,10 +122,10 @@ describe('parser', () => { file: { value: fileContent, options: { - header: { - mtime: fileMtime, - mode: fileMode - } + header: { + mtime: fileMtime, + mode: fileMode + } } } } From 8b569f4256d31782b4fa3fbde863e4e25bcc288b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 23 Dec 2019 15:55:17 +0000 Subject: [PATCH 6/8] feat: store mtime as timespec --- package.json | 2 +- src/parser.js | 8 +++++++- test/parser.spec.js | 12 +++++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index cedb820..f7af9cf 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "devDependencies": { "aegir": "^20.0.0", "chai": "^4.2.0", - "ipfs-http-client": "ipfs/js-ipfs-http-client#support-unixfs-metadata", + "ipfs-http-client": "^40.1.0", "request": "^2.88.0" }, "engines": { diff --git a/src/parser.js b/src/parser.js index 7b23d4b..922ee33 100644 --- a/src/parser.js +++ b/src/parser.js @@ -69,7 +69,13 @@ async function * parseEntry (stream, options) { const entry = {} if (part.headers.mtime) { - entry.mtime = parseInt(part.headers.mtime, 10) + entry.mtime = { + secs: parseInt(part.headers.mtime, 10) + } + + if (part.headers['mtime-nsecs']) { + entry.mtime.nsecs = parseInt(part.headers['mtime-nsecs'], 10) + } } if (part.headers.mode) { diff --git a/test/parser.spec.js b/test/parser.spec.js index 1a79072..e12682b 100644 --- a/test/parser.spec.js +++ b/test/parser.spec.js @@ -27,7 +27,7 @@ const readDir = (path, prefix, includeMetadata, output = []) => { output.push({ path: `${prefix}/${entry}`, - mtime: includeMetadata ? parseInt(type.mtimeMs / 1000) : undefined, + mtime: includeMetadata ? new Date(type.mtimeMs) : undefined, mode: includeMetadata ? type.mode : undefined }) } @@ -36,7 +36,7 @@ const readDir = (path, prefix, includeMetadata, output = []) => { output.push({ path: `${prefix}/${entry}`, content: fs.createReadStream(entryPath), - mtime: includeMetadata ? parseInt(type.mtimeMs / 1000) : undefined, + mtime: includeMetadata ? new Date(type.mtimeMs) : undefined, mode: includeMetadata ? type.mode : undefined }) } @@ -191,9 +191,15 @@ describe('parser', () => { expect(files).to.have.lengthOf(contents.length) for (let i = 0; i < contents.length; i++) { + const msecs = contents[i].mtime.getTime() + const secs = Math.floor(msecs / 1000) + expect(files[i].name).to.equal(contents[i].path) expect(files[i].mode).to.equal(contents[i].mode) - expect(files[i].mtime).to.equal(contents[i].mtime) + expect(files[i].mtime).to.deep.equal({ + secs, + nsecs: (msecs - (secs * 1000)) * 1000 + }) } }) }) From 839692f2f4558babd5311629a810aa0cebd2a355 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 23 Dec 2019 18:22:33 +0000 Subject: [PATCH 7/8] fix: use pr to http-client --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f7af9cf..cedb820 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "devDependencies": { "aegir": "^20.0.0", "chai": "^4.2.0", - "ipfs-http-client": "^40.1.0", + "ipfs-http-client": "ipfs/js-ipfs-http-client#support-unixfs-metadata", "request": "^2.88.0" }, "engines": { From b221aa88b9b206db34941a003800ca1710765e69 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 9 Jan 2020 11:44:50 +0000 Subject: [PATCH 8/8] chore: remove giturl version Update test/parser.spec.js Co-Authored-By: Hugo Dias --- package.json | 2 +- test/parser.spec.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index cedb820..4ba2d37 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "devDependencies": { "aegir": "^20.0.0", "chai": "^4.2.0", - "ipfs-http-client": "ipfs/js-ipfs-http-client#support-unixfs-metadata", + "ipfs-http-client": "^40.2.0", "request": "^2.88.0" }, "engines": { diff --git a/test/parser.spec.js b/test/parser.spec.js index e12682b..fd2f4a4 100644 --- a/test/parser.spec.js +++ b/test/parser.spec.js @@ -117,7 +117,7 @@ describe('parser', () => { request.post({ url: `http://localhost:${PORT}`, formData: formData }, (err) => done(err)) }) - it('parses multipart requests with metatdata correctly', (done) => { + it('parses multipart requests with metadata correctly', (done) => { const formData = { file: { value: fileContent,