feat: support UnixFSv1.5 metadata (#1186)

* feat: support UnixFSv1.5 metadata

* fix: expose new mfs functions

* refactor: send mtime and mode as headers instead of message parts

* fix: include headers for directories

* chore: update ipfs utils dep version

* chore: update ipfs-utils dep

* fix: stringify mode in browser

* test: add tests for unixfs metadata

* fix: fix up tests etc for optional mtime

package.json

@@ -54,7 +54,7 @@
     "explain-error": "^1.0.4",
     "form-data": "^3.0.0",
     "ipfs-block": "~0.8.1",
-    "ipfs-utils": "^0.4.0",
+    "ipfs-utils": "^0.4.2",
     "ipld-dag-cbor": "~0.15.0",
     "ipld-dag-pb": "^0.18.1",
     "ipld-raw": "^4.0.0",
@@ -84,7 +84,7 @@
     "cross-env": "^6.0.0",
     "detect-node": "^2.0.4",
     "go-ipfs-dep": "^0.4.22",
-    "interface-ipfs-core": "~0.125.0",
+    "interface-ipfs-core": "^0.126.0",
     "ipfsd-ctl": "^1.0.0",
     "ndjson": "^1.5.0",
     "nock": "^11.4.0",

src/add/form-data.browser.js

@@ -2,13 +2,29 @@
 /* eslint-env browser */
 
 const normaliseInput = require('ipfs-utils/src/files/normalise-input')
+const mtimeToObject = require('../lib/mtime-to-object')
 
 exports.toFormData = async input => {
   const files = normaliseInput(input)
   const formData = new FormData()
   let i = 0
 
   for await (const file of files) {
+    const headers = {}
+
+    if (file.mtime !== undefined && file.mtime !== null) {
+      const mtime = mtimeToObject(file.mtime)
+
+      if (mtime) {
+        headers.mtime = mtime.secs
+        headers['mtime-nsecs'] = mtime.nsecs
+      }
+    }
+
+    if (file.mode !== undefined && file.mode !== null) {
+      headers.mode = file.mode.toString(8).padStart(4, '0')
+    }
+
     if (file.content) {
       // In the browser there's _currently_ no streaming upload, buffer up our
       // async iterator chunks and append a big Blob :(
@@ -18,9 +34,13 @@ exports.toFormData = async input => {
         bufs.push(chunk)
       }
 
-      formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), encodeURIComponent(file.path))
+      formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), encodeURIComponent(file.path), {
+        header: headers
+      })
     } else {
-      formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), encodeURIComponent(file.path))
+      formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), encodeURIComponent(file.path), {
+        header: headers
+      })
     }
 
     i++

src/add/form-data.js

@@ -5,13 +5,29 @@ const { Buffer } = require('buffer')
 const toStream = require('it-to-stream')
 const normaliseInput = require('ipfs-utils/src/files/normalise-input')
 const { isElectronRenderer } = require('ipfs-utils/src/env')
+const mtimeToObject = require('../lib/mtime-to-object')
 
 exports.toFormData = async input => {
   const files = normaliseInput(input)
   const formData = new FormData()
   let i = 0
 
   for await (const file of files) {
+    const headers = {}
+
+    if (file.mtime !== undefined && file.mtime !== null) {
+      const mtime = mtimeToObject(file.mtime)
+
+      if (mtime) {
+        headers.mtime = mtime.secs
+        headers['mtime-nsecs'] = mtime.nsecs
+      }
+    }
+
+    if (file.mode !== undefined && file.mode !== null) {
+      headers.mode = file.mode.toString(8).padStart(4, '0')
+    }
+
     if (file.content) {
       // In Node.js, FormData can be passed a stream so no need to buffer
       formData.append(
@@ -26,13 +42,15 @@ exports.toFormData = async input => {
         {
           filepath: encodeURIComponent(file.path),
           contentType: 'application/octet-stream',
-          knownLength: file.content.length // Send Content-Length header if known
+          knownLength: file.content.length, // Send Content-Length header if known
+          header: headers
         }
       )
     } else {
       formData.append(`dir-${i}`, Buffer.alloc(0), {
         filepath: encodeURIComponent(file.path),
-        contentType: 'application/x-directory'
+        contentType: 'application/x-directory',
+        header: headers
       })
     }
 

src/add/index.js

@@ -52,6 +52,16 @@ module.exports = configure(({ ky }) => {
   }
 })
 
-function toCoreInterface ({ name, hash, size }) {
-  return { path: name, hash, size: parseInt(size) }
+function toCoreInterface ({ name, hash, size, mode, mtime }) {
+  const output = {
+    path: name,
+    hash,
+    size: parseInt(size)
+  }
+
+  if (mode !== undefined) {
+    output.mode = parseInt(mode, 8)
+  }
+
+  return output
 }

src/files/chmod.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const configure = require('../lib/configure')
+const modeToString = require('../lib/mode-to-string')
+
+module.exports = configure(({ ky }) => {
+  return function chmod (path, mode, options) {
+    options = options || {}
+
+    const searchParams = new URLSearchParams(options.searchParams)
+    searchParams.append('arg', path)
+    searchParams.append('mode', modeToString(mode))
+    if (options.format) searchParams.set('format', options.format)
+    if (options.flush != null) searchParams.set('flush', options.flush)
+    if (options.hashAlg) searchParams.set('hash', options.hashAlg)
+    if (options.parents != null) searchParams.set('parents', options.parents)
+
+    return ky.post('files/chmod', {
+      timeout: options.timeout,
+      signal: options.signal,
+      headers: options.headers,
+      searchParams
+    }).text()
+  }
+})

src/files/index.js

@@ -8,6 +8,7 @@ module.exports = config => {
   const read = require('./read')(config)
 
   return {
+    chmod: callbackify.variadic(require('./chmod')(config)),
     cp: callbackify.variadic(require('./cp')(config)),
     mkdir: callbackify.variadic(require('./mkdir')(config)),
     flush: callbackify.variadic(require('./flush')(config)),
@@ -19,6 +20,7 @@ module.exports = config => {
     read: callbackify.variadic(concatify(read)),
     readReadableStream: streamify.readable(read),
     readPullStream: pullify.source(read),
+    touch: callbackify.variadic(require('./touch')(config)),
     write: callbackify.variadic(require('./write')(config)),
     mv: callbackify.variadic(require('./mv')(config))
   }

src/files/ls.js

@@ -4,7 +4,7 @@ const CID = require('cids')
 const ndjson = require('iterable-ndjson')
 const toIterable = require('../lib/stream-to-iterable')
 const configure = require('../lib/configure')
-const toCamel = require('../lib/object-to-camel')
+const toCamelWithMetadata = require('../lib/object-to-camel-with-metadata')
 
 module.exports = configure(({ ky }) => {
   return async function * ls (path, options) {
@@ -32,11 +32,12 @@ module.exports = configure(({ ky }) => {
       // go-ipfs does not yet support the "stream" option
       if ('Entries' in result) {
         for (const entry of result.Entries || []) {
-          yield toCamel(entry)
+          yield toCamelWithMetadata(entry)
         }
         return
       }
-      yield toCamel(result)
+
+      yield toCamelWithMetadata(result)
     }
   }
 })

src/files/mkdir.js

@@ -1,10 +1,13 @@
 'use strict'
 
 const configure = require('../lib/configure')
+const modeToString = require('../lib/mode-to-string')
+const mtimeToObject = require('../lib/mtime-to-object')
 
 module.exports = configure(({ ky }) => {
   return (path, options) => {
     options = options || {}
+    const mtime = mtimeToObject(options.mtime)
 
     const searchParams = new URLSearchParams(options.searchParams)
     searchParams.append('arg', path)
@@ -13,6 +16,14 @@ module.exports = configure(({ ky }) => {
     if (options.flush != null) searchParams.set('flush', options.flush)
     if (options.hashAlg) searchParams.set('hash', options.hashAlg)
     if (options.parents != null) searchParams.set('parents', options.parents)
+    if (mtime) {
+      searchParams.set('mtime', mtime.secs)
+
+      if (mtime.nsecs != null) {
+        searchParams.set('mtimeNsecs', mtime.nsecs)
+      }
+    }
+    if (options.mode != null) searchParams.set('mode', modeToString(options.mode))
 
     return ky.post('files/mkdir', {
       timeout: options.timeout,

src/files/stat.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const configure = require('../lib/configure')
-const toCamel = require('../lib/object-to-camel')
+const toCamelWithMetadata = require('../lib/object-to-camel-with-metadata')
 
 module.exports = configure(({ ky }) => {
   return async (path, options) => {
@@ -27,6 +27,7 @@ module.exports = configure(({ ky }) => {
     }).json()
 
     res.WithLocality = res.WithLocality || false
-    return toCamel(res)
+
+    return toCamelWithMetadata(res)
   }
 })

src/files/touch.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const configure = require('../lib/configure')
+const mtimeToObject = require('../lib/mtime-to-object')
+
+module.exports = configure(({ ky }) => {
+  return function touch (path, options) {
+    options = options || {}
+    const mtime = mtimeToObject(options.mtime)
+
+    const searchParams = new URLSearchParams(options.searchParams)
+    searchParams.append('arg', path)
+    if (mtime) {
+      searchParams.set('mtime', mtime.secs)
+      searchParams.set('mtimeNsecs', mtime.nsecs)
+    }
+    if (options.format) searchParams.set('format', options.format)
+    if (options.flush != null) searchParams.set('flush', options.flush)
+    if (options.hashAlg) searchParams.set('hash', options.hashAlg)
+    if (options.parents != null) searchParams.set('parents', options.parents)
+
+    return ky.post('files/touch', {
+      timeout: options.timeout,
+      signal: options.signal,
+      headers: options.headers,
+      searchParams
+    }).text()
+  }
+})

src/files/write.js

@@ -2,10 +2,13 @@
 
 const configure = require('../lib/configure')
 const toFormData = require('../lib/buffer-to-form-data')
+const modeToString = require('../lib/mode-to-string')
+const mtimeToObject = require('../lib/mtime-to-object')
 
 module.exports = configure(({ ky }) => {
   return async (path, input, options) => {
     options = options || {}
+    const mtime = mtimeToObject(options.mtime)
 
     const searchParams = new URLSearchParams(options.searchParams)
     searchParams.set('arg', path)
@@ -18,13 +21,24 @@ module.exports = configure(({ ky }) => {
     if (options.parents != null) searchParams.set('parents', options.parents)
     if (options.rawLeaves != null) searchParams.set('raw-leaves', options.rawLeaves)
     if (options.truncate != null) searchParams.set('truncate', options.truncate)
+    if (mtime) {
+      searchParams.set('mtime', mtime.secs)
+
+      if (mtime.nsecs != null) {
+        searchParams.set('mtimeNsecs', mtime.nsecs)
+      }
+    }
 
     const res = await ky.post('files/write', {
       timeout: options.timeout,
       signal: options.signal,
       headers: options.headers,
       searchParams,
-      body: toFormData(input) // TODO: support inputs other than buffer as per spec
+      body: toFormData(input, {
+        mode: options.mode != null ? modeToString(options.mode) : undefined,
+        mtime: mtime ? mtime.secs : undefined,
+        mtimeNsecs: mtime ? mtime.nsecs : undefined
+      }) // TODO: support inputs other than buffer as per spec
     })
 
     return res.text()

src/lib/buffer-to-form-data.js

@@ -3,9 +3,25 @@
 const FormData = require('form-data')
 const { isElectronRenderer } = require('ipfs-utils/src/env')
 
-module.exports = buf => {
+module.exports = (buf, { mode, mtime, mtimeNsecs } = {}) => {
+  const headers = {}
+
+  if (mode != null) {
+    headers.mode = mode
+  }
+
+  if (mtime != null) {
+    headers.mtime = mtime
+
+    if (mtimeNsecs != null) {
+      headers['mtime-nsecs'] = mtimeNsecs
+    }
+  }
+
   const formData = new FormData()
-  formData.append('file', buf)
+  formData.append('file', buf, {
+    header: headers
+  })
   return formData
 }
 

src/lib/mode-to-string.js

@@ -0,0 +1,13 @@
+'use strict'
+
+module.exports = (mode) => {
+  if (mode === undefined || mode === null) {
+    return undefined
+  }
+
+  if (typeof mode === 'string' || mode instanceof String) {
+    return mode
+  }
+
+  return mode.toString(8).padStart(4, '0')
+}