Skip to content
This repository has been archived by the owner on Apr 29, 2020. It is now read-only.

Commit

Permalink
feat: support UnixFSv1.5 metadata
Browse files Browse the repository at this point in the history 
* refactor: pass mode and mtime in headers

* refactor: refactor clunky test

* feat: store mtime as timespec
  • Loading branch information
@achingbrain @hugomrdias
achingbrain authored and hugomrdias committed Jan 9, 2020
1 parent 34007f4 commit 008e872
Show file tree
Hide file tree
Showing 3 changed files with 148 additions and 65 deletions.
4 changes: 2 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@
},
"dependencies": {
"@hapi/content": "^4.1.0",
"it-multipart": "~0.0.2"
"it-multipart": "^1.0.1"
},
"devDependencies": {
"aegir": "^20.0.0",
"chai": "^4.2.0",
"ipfs-http-client": "^35.1.0",
"ipfs-http-client": "^40.2.0",
"request": "^2.88.0"
},
"engines": {
Expand Down
121 changes: 79 additions & 42 deletions src/parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,16 @@ const isDirectory = (mediatype) => mediatype === multipartFormdataType || mediat
const parseDisposition = (disposition) => {
const details = {}
details.type = disposition.split(';')[0]
if (details.type === 'file' || details.type === 'form-data') {
const namePattern = / filename="(.[^"]+)"/
const matches = disposition.match(namePattern)
details.name = matches ? matches[1] : ''
}

return details
}

const parseHeader = (header) => {
const type = Content.type(header['content-type'])
const disposition = parseDisposition(header['content-disposition'])
if (details.type === 'file' || details.type === 'form-data') {
const filenamePattern = / filename="(.[^"]+)"/
const filenameMatches = disposition.match(filenamePattern)
details.filename = filenameMatches ? filenameMatches[1] : ''

const details = type
details.name = decodeURIComponent(disposition.name)
details.type = disposition.type
const namePattern = / name="(.[^"]+)"/
const nameMatches = disposition.match(namePattern)
details.name = nameMatches ? nameMatches[1] : ''
}

return details
}
Expand All @@ -50,49 +44,92 @@ const ignore = async (stream) => {
}
}

async function * parser (stream, options) {
for await (const part of multipart(stream, options.boundary)) {
const partHeader = parseHeader(part.headers)
async function * parseEntry (stream, options) {
for await (const part of stream) {
if (!part.headers['content-type']) {
throw new Error('No content-type in multipart part')
}

if (isDirectory(partHeader.mime)) {
yield {
type: 'directory',
name: partHeader.name
}
const type = Content.type(part.headers['content-type'])

await ignore(part.body)
if (type.boundary) {
// recursively parse nested multiparts
yield * parser(part.body, {
...options,
boundary: type.boundary
})

continue
}

if (partHeader.mime === applicationSymlink) {
const target = await collect(part.body)
if (!part.headers['content-disposition']) {
throw new Error('No content disposition in multipart part')
}

yield {
type: 'symlink',
name: partHeader.name,
target: target.toString('utf8')
const entry = {}

if (part.headers.mtime) {
entry.mtime = {
secs: parseInt(part.headers.mtime, 10)
}

continue
if (part.headers['mtime-nsecs']) {
entry.mtime.nsecs = parseInt(part.headers['mtime-nsecs'], 10)
}
}

if (partHeader.boundary) {
// recursively parse nested multiparts
for await (const entry of parser(part, {
...options,
boundary: partHeader.boundary
})) {
yield entry
if (part.headers.mode) {
entry.mode = parseInt(part.headers.mode, 8)
}

if (isDirectory(type.mime)) {
entry.type = 'directory'
} else if (type.mime === applicationSymlink) {
entry.type = 'symlink'
} else {
entry.type = 'file'
}

const disposition = parseDisposition(part.headers['content-disposition'])

entry.name = decodeURIComponent(disposition.filename)
entry.body = part.body

yield entry
}
}

async function * parser (stream, options) {
for await (const entry of parseEntry(multipart(stream, options.boundary), options)) {
if (entry.type === 'directory') {
yield {
type: 'directory',
name: entry.name,
mtime: entry.mtime,
mode: entry.mode
}

continue
await ignore(entry.body)
}

if (entry.type === 'symlink') {
yield {
type: 'symlink',
name: entry.name,
target: (await collect(entry.body)).toString('utf8'),
mtime: entry.mtime,
mode: entry.mode
}
}

yield {
type: 'file',
name: partHeader.name,
content: part.body
if (entry.type === 'file') {
yield {
type: 'file',
name: entry.name,
content: entry.body,
mtime: entry.mtime,
mode: entry.mode
}
}
}
}
Expand Down
88 changes: 67 additions & 21 deletions test/parser.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ const os = require('os')

const isWindows = os.platform() === 'win32'

const readDir = (path, prefix, output = []) => {
const readDir = (path, prefix, includeMetadata, output = []) => {
const entries = fs.readdirSync(path)

entries.forEach(entry => {
Expand All @@ -23,21 +23,25 @@ const readDir = (path, prefix, output = []) => {
const type = fs.statSync(entryPath)

if (type.isDirectory()) {
readDir(entryPath, `${prefix}/${entry}`, output)
readDir(entryPath, `${prefix}/${entry}`, includeMetadata, output)

output.push({
path: `${prefix}/${entry}`,
mtime: includeMetadata ? new Date(type.mtimeMs) : undefined,
mode: includeMetadata ? type.mode : undefined
})
}

if (type.isFile()) {
output.push({
path: `${prefix}/${entry}`,
content: fs.createReadStream(entryPath)
content: fs.createReadStream(entryPath),
mtime: includeMetadata ? new Date(type.mtimeMs) : undefined,
mode: includeMetadata ? type.mode : undefined
})
}
})

output.push({
path: prefix
})

return output
}

Expand Down Expand Up @@ -75,6 +79,8 @@ describe('parser', () => {
describe('single file', () => {
const filePath = path.resolve(__dirname, 'fixtures/config')
const fileContent = fs.readFileSync(filePath, 'utf8')
const fileMtime = parseInt(Date.now() / 1000)
const fileMode = parseInt('0777', 8)

before(() => {
handler = async (req) => {
Expand All @@ -84,7 +90,7 @@ describe('parser', () => {

for await (const entry of parser(req)) {
if (entry.type === 'file') {
const file = { name: entry.name, content: '' }
const file = { ...entry, content: '' }

for await (const data of entry.content) {
file.content += data.toString()
Expand All @@ -95,13 +101,12 @@ describe('parser', () => {
}

expect(files.length).to.equal(1)
expect(files[0].name).to.equal('config')
expect(files[0].content).to.equal(fileContent)
expect(JSON.parse(files[0].content)).to.deep.equal(JSON.parse(fileContent))
}
})

it('parses ctl.config.replace correctly', async () => {
await ctl.config.replace(filePath)
await ctl.config.replace(JSON.parse(fileContent))
})

it('parses regular multipart requests correctly', (done) => {
Expand All @@ -111,6 +116,22 @@ describe('parser', () => {

request.post({ url: `http://localhost:${PORT}`, formData: formData }, (err) => done(err))
})

it('parses multipart requests with metadata correctly', (done) => {
const formData = {
file: {
value: fileContent,
options: {
header: {
mtime: fileMtime,
mode: fileMode
}
}
}
}

request.post({ url: `http://localhost:${PORT}`, formData }, (err) => done(err))
})
})

describe('directory', () => {
Expand All @@ -123,15 +144,15 @@ describe('parser', () => {
expect(req.headers['content-type']).to.be.a('string')

for await (const entry of parser(req)) {
if (entry.type === 'file') {
const file = { name: entry.name, content: '' }
const file = { ...entry, content: '' }

if (entry.content) {
for await (const data of entry.content) {
file.content += data.toString()
}

files.push(file)
}

files.push(file)
}
}
})
Expand All @@ -149,12 +170,37 @@ describe('parser', () => {
return
}

expect(files.length).to.equal(5)
expect(files[0].name).to.equal('fixtures/config')
expect(files[1].name).to.equal('fixtures/folderlink/deepfile')
expect(files[2].name).to.equal('fixtures/link')
expect(files[3].name).to.equal('fixtures/otherfile')
expect(files[4].name).to.equal('fixtures/subfolder/deepfile')
expect(files).to.have.lengthOf(contents.length)

for (let i = 0; i < contents.length; i++) {
expect(files[i].name).to.equal(contents[i].path)
expect(files[i].mode).to.be.undefined
expect(files[i].mtime).to.be.undefined
}
})

it('parses ctl.add with metadata correctly', async () => {
const contents = readDir(dirPath, 'fixtures', true)

await ctl.add(contents, { recursive: true, followSymlinks: false })

if (isWindows) {
return
}

expect(files).to.have.lengthOf(contents.length)

for (let i = 0; i < contents.length; i++) {
const msecs = contents[i].mtime.getTime()
const secs = Math.floor(msecs / 1000)

expect(files[i].name).to.equal(contents[i].path)
expect(files[i].mode).to.equal(contents[i].mode)
expect(files[i].mtime).to.deep.equal({
secs,
nsecs: (msecs - (secs * 1000)) * 1000
})
}
})
})

Expand Down

0 comments on commit 008e872

Please sign in to comment.