From da9d17a38ce09d299e7180d489a56c1e276b4fb9 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 9 Jan 2020 11:04:33 +0000 Subject: [PATCH] feat: support UnixFSv1.5 metadata (#1186) * feat: support UnixFSv1.5 metadata * fix: expose new mfs functions * refactor: send mtime and mode as headers instead of message parts * fix: include headers for directories * chore: update ipfs utils dep version * chore: update ipfs-utils dep * fix: stringify mode in browser * test: add tests for unixfs metadata * fix: fix up tests etc for optional mtime --- package.json | 4 +- src/add/form-data.browser.js | 24 +++- src/add/form-data.js | 22 +++- src/add/index.js | 14 +- src/files/chmod.js | 25 ++++ src/files/index.js | 2 + src/files/ls.js | 7 +- src/files/mkdir.js | 11 ++ src/files/stat.js | 5 +- src/files/touch.js | 29 +++++ src/files/write.js | 16 ++- src/lib/buffer-to-form-data.js | 20 ++- src/lib/mode-to-string.js | 13 ++ src/lib/mtime-to-object.js | 56 ++++++++ src/lib/object-to-camel-with-metadata.js | 24 ++++ src/ls.js | 18 ++- test/interface.spec.js | 156 +++++++++++++++++++++++ 17 files changed, 429 insertions(+), 17 deletions(-) create mode 100644 src/files/chmod.js create mode 100644 src/files/touch.js create mode 100644 src/lib/mode-to-string.js create mode 100644 src/lib/mtime-to-object.js create mode 100644 src/lib/object-to-camel-with-metadata.js diff --git a/package.json b/package.json index f0253c933..261b888c9 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "explain-error": "^1.0.4", "form-data": "^3.0.0", "ipfs-block": "~0.8.1", - "ipfs-utils": "^0.4.0", + "ipfs-utils": "^0.4.2", "ipld-dag-cbor": "~0.15.0", "ipld-dag-pb": "^0.18.1", "ipld-raw": "^4.0.0", @@ -84,7 +84,7 @@ "cross-env": "^6.0.0", "detect-node": "^2.0.4", "go-ipfs-dep": "^0.4.22", - "interface-ipfs-core": "~0.125.0", + "interface-ipfs-core": "^0.126.0", "ipfsd-ctl": "^1.0.0", "ndjson": "^1.5.0", "nock": "^11.4.0", diff --git a/src/add/form-data.browser.js b/src/add/form-data.browser.js index 247396c42..484f67c6c 100644 --- a/src/add/form-data.browser.js +++ b/src/add/form-data.browser.js @@ -2,6 +2,7 @@ /* eslint-env browser */ const normaliseInput = require('ipfs-utils/src/files/normalise-input') +const mtimeToObject = require('../lib/mtime-to-object') exports.toFormData = async input => { const files = normaliseInput(input) @@ -9,6 +10,21 @@ exports.toFormData = async input => { let i = 0 for await (const file of files) { + const headers = {} + + if (file.mtime !== undefined && file.mtime !== null) { + const mtime = mtimeToObject(file.mtime) + + if (mtime) { + headers.mtime = mtime.secs + headers['mtime-nsecs'] = mtime.nsecs + } + } + + if (file.mode !== undefined && file.mode !== null) { + headers.mode = file.mode.toString(8).padStart(4, '0') + } + if (file.content) { // In the browser there's _currently_ no streaming upload, buffer up our // async iterator chunks and append a big Blob :( @@ -18,9 +34,13 @@ exports.toFormData = async input => { bufs.push(chunk) } - formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), encodeURIComponent(file.path)) + formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), encodeURIComponent(file.path), { + header: headers + }) } else { - formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), encodeURIComponent(file.path)) + formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), encodeURIComponent(file.path), { + header: headers + }) } i++ diff --git a/src/add/form-data.js b/src/add/form-data.js index 96d55f672..1ce5050da 100644 --- a/src/add/form-data.js +++ b/src/add/form-data.js @@ -5,6 +5,7 @@ const { Buffer } = require('buffer') const toStream = require('it-to-stream') const normaliseInput = require('ipfs-utils/src/files/normalise-input') const { isElectronRenderer } = require('ipfs-utils/src/env') +const mtimeToObject = require('../lib/mtime-to-object') exports.toFormData = async input => { const files = normaliseInput(input) @@ -12,6 +13,21 @@ exports.toFormData = async input => { let i = 0 for await (const file of files) { + const headers = {} + + if (file.mtime !== undefined && file.mtime !== null) { + const mtime = mtimeToObject(file.mtime) + + if (mtime) { + headers.mtime = mtime.secs + headers['mtime-nsecs'] = mtime.nsecs + } + } + + if (file.mode !== undefined && file.mode !== null) { + headers.mode = file.mode.toString(8).padStart(4, '0') + } + if (file.content) { // In Node.js, FormData can be passed a stream so no need to buffer formData.append( @@ -26,13 +42,15 @@ exports.toFormData = async input => { { filepath: encodeURIComponent(file.path), contentType: 'application/octet-stream', - knownLength: file.content.length // Send Content-Length header if known + knownLength: file.content.length, // Send Content-Length header if known + header: headers } ) } else { formData.append(`dir-${i}`, Buffer.alloc(0), { filepath: encodeURIComponent(file.path), - contentType: 'application/x-directory' + contentType: 'application/x-directory', + header: headers }) } diff --git a/src/add/index.js b/src/add/index.js index df6afbad3..546e5428d 100644 --- a/src/add/index.js +++ b/src/add/index.js @@ -52,6 +52,16 @@ module.exports = configure(({ ky }) => { } }) -function toCoreInterface ({ name, hash, size }) { - return { path: name, hash, size: parseInt(size) } +function toCoreInterface ({ name, hash, size, mode, mtime }) { + const output = { + path: name, + hash, + size: parseInt(size) + } + + if (mode !== undefined) { + output.mode = parseInt(mode, 8) + } + + return output } diff --git a/src/files/chmod.js b/src/files/chmod.js new file mode 100644 index 000000000..b4c0a11dd --- /dev/null +++ b/src/files/chmod.js @@ -0,0 +1,25 @@ +'use strict' + +const configure = require('../lib/configure') +const modeToString = require('../lib/mode-to-string') + +module.exports = configure(({ ky }) => { + return function chmod (path, mode, options) { + options = options || {} + + const searchParams = new URLSearchParams(options.searchParams) + searchParams.append('arg', path) + searchParams.append('mode', modeToString(mode)) + if (options.format) searchParams.set('format', options.format) + if (options.flush != null) searchParams.set('flush', options.flush) + if (options.hashAlg) searchParams.set('hash', options.hashAlg) + if (options.parents != null) searchParams.set('parents', options.parents) + + return ky.post('files/chmod', { + timeout: options.timeout, + signal: options.signal, + headers: options.headers, + searchParams + }).text() + } +}) diff --git a/src/files/index.js b/src/files/index.js index 25e79fcab..ee6b7d8d8 100644 --- a/src/files/index.js +++ b/src/files/index.js @@ -8,6 +8,7 @@ module.exports = config => { const read = require('./read')(config) return { + chmod: callbackify.variadic(require('./chmod')(config)), cp: callbackify.variadic(require('./cp')(config)), mkdir: callbackify.variadic(require('./mkdir')(config)), flush: callbackify.variadic(require('./flush')(config)), @@ -19,6 +20,7 @@ module.exports = config => { read: callbackify.variadic(concatify(read)), readReadableStream: streamify.readable(read), readPullStream: pullify.source(read), + touch: callbackify.variadic(require('./touch')(config)), write: callbackify.variadic(require('./write')(config)), mv: callbackify.variadic(require('./mv')(config)) } diff --git a/src/files/ls.js b/src/files/ls.js index 1baa3f656..51ee33912 100644 --- a/src/files/ls.js +++ b/src/files/ls.js @@ -4,7 +4,7 @@ const CID = require('cids') const ndjson = require('iterable-ndjson') const toIterable = require('../lib/stream-to-iterable') const configure = require('../lib/configure') -const toCamel = require('../lib/object-to-camel') +const toCamelWithMetadata = require('../lib/object-to-camel-with-metadata') module.exports = configure(({ ky }) => { return async function * ls (path, options) { @@ -32,11 +32,12 @@ module.exports = configure(({ ky }) => { // go-ipfs does not yet support the "stream" option if ('Entries' in result) { for (const entry of result.Entries || []) { - yield toCamel(entry) + yield toCamelWithMetadata(entry) } return } - yield toCamel(result) + + yield toCamelWithMetadata(result) } } }) diff --git a/src/files/mkdir.js b/src/files/mkdir.js index 0fc3c238d..3a50c7728 100644 --- a/src/files/mkdir.js +++ b/src/files/mkdir.js @@ -1,10 +1,13 @@ 'use strict' const configure = require('../lib/configure') +const modeToString = require('../lib/mode-to-string') +const mtimeToObject = require('../lib/mtime-to-object') module.exports = configure(({ ky }) => { return (path, options) => { options = options || {} + const mtime = mtimeToObject(options.mtime) const searchParams = new URLSearchParams(options.searchParams) searchParams.append('arg', path) @@ -13,6 +16,14 @@ module.exports = configure(({ ky }) => { if (options.flush != null) searchParams.set('flush', options.flush) if (options.hashAlg) searchParams.set('hash', options.hashAlg) if (options.parents != null) searchParams.set('parents', options.parents) + if (mtime) { + searchParams.set('mtime', mtime.secs) + + if (mtime.nsecs != null) { + searchParams.set('mtimeNsecs', mtime.nsecs) + } + } + if (options.mode != null) searchParams.set('mode', modeToString(options.mode)) return ky.post('files/mkdir', { timeout: options.timeout, diff --git a/src/files/stat.js b/src/files/stat.js index 98026283e..1b4af061b 100644 --- a/src/files/stat.js +++ b/src/files/stat.js @@ -1,7 +1,7 @@ 'use strict' const configure = require('../lib/configure') -const toCamel = require('../lib/object-to-camel') +const toCamelWithMetadata = require('../lib/object-to-camel-with-metadata') module.exports = configure(({ ky }) => { return async (path, options) => { @@ -27,6 +27,7 @@ module.exports = configure(({ ky }) => { }).json() res.WithLocality = res.WithLocality || false - return toCamel(res) + + return toCamelWithMetadata(res) } }) diff --git a/src/files/touch.js b/src/files/touch.js new file mode 100644 index 000000000..b38aca905 --- /dev/null +++ b/src/files/touch.js @@ -0,0 +1,29 @@ +'use strict' + +const configure = require('../lib/configure') +const mtimeToObject = require('../lib/mtime-to-object') + +module.exports = configure(({ ky }) => { + return function touch (path, options) { + options = options || {} + const mtime = mtimeToObject(options.mtime) + + const searchParams = new URLSearchParams(options.searchParams) + searchParams.append('arg', path) + if (mtime) { + searchParams.set('mtime', mtime.secs) + searchParams.set('mtimeNsecs', mtime.nsecs) + } + if (options.format) searchParams.set('format', options.format) + if (options.flush != null) searchParams.set('flush', options.flush) + if (options.hashAlg) searchParams.set('hash', options.hashAlg) + if (options.parents != null) searchParams.set('parents', options.parents) + + return ky.post('files/touch', { + timeout: options.timeout, + signal: options.signal, + headers: options.headers, + searchParams + }).text() + } +}) diff --git a/src/files/write.js b/src/files/write.js index 77a772ea6..e31f0a8a2 100644 --- a/src/files/write.js +++ b/src/files/write.js @@ -2,10 +2,13 @@ const configure = require('../lib/configure') const toFormData = require('../lib/buffer-to-form-data') +const modeToString = require('../lib/mode-to-string') +const mtimeToObject = require('../lib/mtime-to-object') module.exports = configure(({ ky }) => { return async (path, input, options) => { options = options || {} + const mtime = mtimeToObject(options.mtime) const searchParams = new URLSearchParams(options.searchParams) searchParams.set('arg', path) @@ -18,13 +21,24 @@ module.exports = configure(({ ky }) => { if (options.parents != null) searchParams.set('parents', options.parents) if (options.rawLeaves != null) searchParams.set('raw-leaves', options.rawLeaves) if (options.truncate != null) searchParams.set('truncate', options.truncate) + if (mtime) { + searchParams.set('mtime', mtime.secs) + + if (mtime.nsecs != null) { + searchParams.set('mtimeNsecs', mtime.nsecs) + } + } const res = await ky.post('files/write', { timeout: options.timeout, signal: options.signal, headers: options.headers, searchParams, - body: toFormData(input) // TODO: support inputs other than buffer as per spec + body: toFormData(input, { + mode: options.mode != null ? modeToString(options.mode) : undefined, + mtime: mtime ? mtime.secs : undefined, + mtimeNsecs: mtime ? mtime.nsecs : undefined + }) // TODO: support inputs other than buffer as per spec }) return res.text() diff --git a/src/lib/buffer-to-form-data.js b/src/lib/buffer-to-form-data.js index 41f03383e..1a4830361 100644 --- a/src/lib/buffer-to-form-data.js +++ b/src/lib/buffer-to-form-data.js @@ -3,9 +3,25 @@ const FormData = require('form-data') const { isElectronRenderer } = require('ipfs-utils/src/env') -module.exports = buf => { +module.exports = (buf, { mode, mtime, mtimeNsecs } = {}) => { + const headers = {} + + if (mode != null) { + headers.mode = mode + } + + if (mtime != null) { + headers.mtime = mtime + + if (mtimeNsecs != null) { + headers['mtime-nsecs'] = mtimeNsecs + } + } + const formData = new FormData() - formData.append('file', buf) + formData.append('file', buf, { + header: headers + }) return formData } diff --git a/src/lib/mode-to-string.js b/src/lib/mode-to-string.js new file mode 100644 index 000000000..ee2742b9a --- /dev/null +++ b/src/lib/mode-to-string.js @@ -0,0 +1,13 @@ +'use strict' + +module.exports = (mode) => { + if (mode === undefined || mode === null) { + return undefined + } + + if (typeof mode === 'string' || mode instanceof String) { + return mode + } + + return mode.toString(8).padStart(4, '0') +} diff --git a/src/lib/mtime-to-object.js b/src/lib/mtime-to-object.js new file mode 100644 index 000000000..be89148f6 --- /dev/null +++ b/src/lib/mtime-to-object.js @@ -0,0 +1,56 @@ +'use strict' + +module.exports = function parseMtime (mtime) { + if (mtime == null) { + return undefined + } + + // Javascript Date + if (mtime instanceof Date) { + const ms = mtime.getTime() + const secs = Math.floor(ms / 1000) + + return { + secs: secs, + nsecs: (ms - (secs * 1000)) * 1000 + } + } + + // { secs, nsecs } + if (Object.prototype.hasOwnProperty.call(mtime, 'secs')) { + return { + secs: mtime.secs, + nsecs: mtime.nsecs + } + } + + // UnixFS TimeSpec + if (Object.prototype.hasOwnProperty.call(mtime, 'Seconds')) { + return { + secs: mtime.Seconds, + nsecs: mtime.FractionalNanoseconds + } + } + + // process.hrtime() + if (Array.isArray(mtime)) { + return { + secs: mtime[0], + nsecs: mtime[1] + } + } + /* + TODO: https://github.com/ipfs/aegir/issues/487 + + // process.hrtime.bigint() + if (typeof mtime === 'bigint') { + const secs = mtime / BigInt(1e9) + const nsecs = mtime - (secs * BigInt(1e9)) + + return { + secs: parseInt(secs), + nsecs: parseInt(nsecs) + } + } + */ +} diff --git a/src/lib/object-to-camel-with-metadata.js b/src/lib/object-to-camel-with-metadata.js new file mode 100644 index 000000000..55f16d0bb --- /dev/null +++ b/src/lib/object-to-camel-with-metadata.js @@ -0,0 +1,24 @@ +'use strict' + +const toCamel = require('./object-to-camel') + +function toCamelWithMetadata (entry) { + const file = toCamel(entry) + + if (Object.prototype.hasOwnProperty.call(file, 'mode')) { + file.mode = parseInt(file.mode, 8) + } + + if (Object.prototype.hasOwnProperty.call(file, 'mtime')) { + file.mtime = { + secs: file.mtime, + nsecs: file.mtimeNsecs || 0 + } + + delete file.mtimeNsecs + } + + return file +} + +module.exports = toCamelWithMetadata diff --git a/src/ls.js b/src/ls.js index a9cd476f9..43e92a54a 100644 --- a/src/ls.js +++ b/src/ls.js @@ -48,7 +48,7 @@ module.exports = configure(({ ky }) => { } for (const link of result) { - yield { + const entry = { name: link.Name, path: path + '/' + link.Name, size: link.Size, @@ -56,6 +56,22 @@ module.exports = configure(({ ky }) => { type: typeOf(link), depth: link.Depth || 1 } + + if (link.Mode) { + entry.mode = parseInt(link.Mode, 8) + } + + if (link.Mtime !== undefined && link.Mtime !== null) { + entry.mtime = { + secs: link.Mtime + } + + if (link.MtimeNsecs !== undefined && link.MtimeNsecs !== null) { + entry.mtime.nsecs = link.MtimeNsecs + } + } + + yield entry } } }) diff --git a/test/interface.spec.js b/test/interface.spec.js index 5435518cf..e09cac5a6 100644 --- a/test/interface.spec.js +++ b/test/interface.spec.js @@ -114,6 +114,134 @@ describe('interface-ipfs-core tests', () => { { name: 'should ls from outside of mfs', reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should change file mode', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should change directory mode', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should change file mode as string', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should change file mode to 0', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should update file mtime', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should update directory mtime', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should make directory and specify mode', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should make directory and specify mtime', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should write file and specify mode', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should write file and specify mtime', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should respect metadata when copying files', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should respect metadata when copying directories', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should respect metadata when copying from outside of mfs', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'ls directory with long option should include metadata', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should have default mtime', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should set mtime as Date', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should set mtime as { nsecs, secs }', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should set mtime as timespec', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should set mtime as hrtime', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should make directory and have default mode', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should make directory and specify mode as string', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should make directory and specify mode as number', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should make directory and specify mtime as Date', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should make directory and specify mtime as { nsecs, secs }', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should make directory and specify mtime as timespec', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should make directory and specify mtime as hrtime', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should write file and specify mode as a string', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should write file and specify mode as a number', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should write file and specify mtime as Date', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should write file and specify mtime as { nsecs, secs }', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should write file and specify mtime as timespec', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should write file and specify mtime as hrtime', + reason: 'TODO not implemented in go-ipfs yet' } ] }) @@ -125,6 +253,30 @@ describe('interface-ipfs-core tests', () => { name: 'addFromFs', reason: 'Not designed to run in the browser' }, + { + name: 'should add with mode as string', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should add with mode as number', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should add with mtime as Date', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should add with mtime as { nsecs, secs }', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should add with mtime as timespec', + reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should add with mtime as hrtime', + reason: 'TODO not implemented in go-ipfs yet' + }, // .catPullStream { name: 'should export a chunk of a file', @@ -137,6 +289,10 @@ describe('interface-ipfs-core tests', () => { { name: 'should export a chunk of a file in a Readable Stream', reason: 'TODO not implemented in go-ipfs yet' + }, + { + name: 'should ls with metadata', + reason: 'TODO not implemented in go-ipfs yet' } ] })