refactor: use unixfs exporter with CID instances (#1902)

Allows IPFS to use the unixfs exporter that exports CID instances _without_ API changes.

Next step is to get the unixfs importer to do the same and then make a breaking interface JS IPFS core API change to ensure we're always returning CID instances not strings for add, ls etc etc.

refs ipfs-inactive/interface-js-ipfs-core#394

Depends on:

* [x] ipfs-inactive/js-ipfs-mfs#44
* [x] ipfs-inactive/js-ipfs-unixfs-exporter#19

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>

package.json

@@ -108,11 +108,12 @@
     "ipfs-block-service": "~0.15.1",
     "ipfs-http-client": "^30.0.0",
     "ipfs-http-response": "~0.2.1",
-    "ipfs-mfs": "~0.9.1",
+    "ipfs-mfs": "~0.10.2",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.26.1",
     "ipfs-unixfs": "~0.1.16",
-    "ipfs-unixfs-engine": "~0.35.3",
+    "ipfs-unixfs-exporter": "~0.36.1",
+    "ipfs-unixfs-importer": "~0.38.5",
     "ipld": "~0.21.1",
     "ipld-bitcoin": "~0.1.8",
     "ipld-dag-pb": "~0.15.3",

src/cli/commands/ls.js

@@ -59,7 +59,7 @@ module.exports = {
         const padding = link.depth - pathParts.length
         print(
           rightpad(link.hash, multihashWidth + 1) +
-          rightpad(link.size || '', sizeWidth + 1) +
+          rightpad(link.size || '-', sizeWidth + 1) +
           '  '.repeat(padding) + fileName
         )
       })

src/core/components/files-regular/add-pull-stream.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { importer } = require('ipfs-unixfs-engine')
+const importer = require('ipfs-unixfs-importer')
 const pull = require('pull-stream')
 const toPull = require('stream-to-pull-stream')
 const waterfall = require('async/waterfall')

src/core/components/files-regular/cat-pull-stream.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { exporter } = require('ipfs-unixfs-engine')
+const exporter = require('ipfs-unixfs-exporter')
 const pull = require('pull-stream')
 const deferred = require('pull-defer')
 const { normalizePath } = require('./utils')

src/core/components/files-regular/get-pull-stream.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { exporter } = require('ipfs-unixfs-engine')
+const exporter = require('ipfs-unixfs-exporter')
 const pull = require('pull-stream')
 const errCode = require('err-code')
 const { normalizePath } = require('./utils')
@@ -21,6 +21,13 @@ module.exports = function (self) {
       self._preload(pathComponents[0])
     }
 
-    return exporter(ipfsPath, self._ipld, options)
+    return pull(
+      exporter(ipfsPath, self._ipld, options),
+      pull.map(file => {
+        file.hash = file.cid.toString()
+        delete file.cid
+        return file
+      })
+    )
   }
 }

src/core/components/files-regular/ls-pull-stream.js

@@ -1,8 +1,7 @@
 'use strict'
 
-const { exporter } = require('ipfs-unixfs-engine')
+const exporter = require('ipfs-unixfs-exporter')
 const pull = require('pull-stream')
-const CID = require('cids')
 const { normalizePath } = require('./utils')
 
 module.exports = function (self) {
@@ -26,7 +25,8 @@ module.exports = function (self) {
         recursive ? node.depth >= pathDepth : node.depth === pathDepth
       ),
       pull.map(node => {
-        node.hash = new CID(node.hash).toBaseEncodedString()
+        node.hash = node.cid.toString()
+        delete node.cid
         delete node.content
         return node
       })

src/http/api/resources/file.js

@@ -1,8 +1,7 @@
 'use strict'
 
 const isIpfs = require('is-ipfs')
-const unixfsEngine = require('ipfs-unixfs-engine')
-const exporter = unixfsEngine.exporter
+const exporter = require('ipfs-unixfs-exporter')
 const pull = require('pull-stream')
 const toB58String = require('multihashes').toB58String
 const Boom = require('boom')
@@ -82,7 +81,7 @@ exports.ls = {
     const links = []
     files.forEach((file) => {
       if (file.depth === rootDepth) {
-        const id = toB58String(file.hash)
+        const id = file.cid.toString()
         res.Arguments[path] = id
         res.Objects[id] = toFileObject(file)
         res.Objects[id].Links = file.type === 'file' ? null : links

test/cli/ls.js

@@ -17,11 +17,11 @@ describe('ls', () => runOnAndOff((thing) => {
     return ipfs('ls Qmaj2NmcyAXT8dFmZRRytE12wpcaHADzbChKToMEjBsj5Z')
       .then((out) => {
         expect(out).to.eql(
-          'QmamKEPmEH9RUsqRQsfNf5evZQDQPYL9KXg1ADeT7mkHkT 123530 blocks/\n' +
-          'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939   config\n' +
-          'QmUqyZtPmsRy1U5Mo8kz2BAMmk1hfJ7yW1KAFTMB2odsFv 5503   datastore/\n' +
-          'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397   init-docs/\n' +
-          'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10     version\n'
+          'QmamKEPmEH9RUsqRQsfNf5evZQDQPYL9KXg1ADeT7mkHkT -    blocks/\n' +
+          'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3928 config\n' +
+          'QmUqyZtPmsRy1U5Mo8kz2BAMmk1hfJ7yW1KAFTMB2odsFv -    datastore/\n' +
+          'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU -    init-docs/\n' +
+          'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 2    version\n'
         )
       })
   })
@@ -41,12 +41,12 @@ describe('ls', () => runOnAndOff((thing) => {
     return ipfs('ls /ipfs/Qmaj2NmcyAXT8dFmZRRytE12wpcaHADzbChKToMEjBsj5Z -v')
       .then((out) => {
         expect(out).to.eql(
-          'Hash                                           Size   Name\n' +
-          'QmamKEPmEH9RUsqRQsfNf5evZQDQPYL9KXg1ADeT7mkHkT 123530 blocks/\n' +
-          'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939   config\n' +
-          'QmUqyZtPmsRy1U5Mo8kz2BAMmk1hfJ7yW1KAFTMB2odsFv 5503   datastore/\n' +
-          'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397   init-docs/\n' +
-          'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10     version\n'
+          'Hash                                           Size Name\n' +
+          'QmamKEPmEH9RUsqRQsfNf5evZQDQPYL9KXg1ADeT7mkHkT -    blocks/\n' +
+          'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3928 config\n' +
+          'QmUqyZtPmsRy1U5Mo8kz2BAMmk1hfJ7yW1KAFTMB2odsFv -    datastore/\n' +
+          'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU -    init-docs/\n' +
+          'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 2    version\n'
         )
       })
   })
@@ -57,14 +57,14 @@ describe('ls', () => runOnAndOff((thing) => {
     return ipfs('ls /ipfs/Qmaj2NmcyAXT8dFmZRRytE12wpcaHADzbChKToMEjBsj5Z/init-docs')
       .then((out) => {
         expect(out).to.eql(
-          'QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V 1688 about\n' +
-          'QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y 200  contact\n' +
-          'QmegvLXxpVKiZ4b57Xs1syfBVRd8CbucVHAp7KpLQdGieC 65   docs/\n' +
-          'QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 322  help\n' +
-          'QmdncfsVm2h5Kqq9hPmU7oAVX2zTSVP3L869tgTbPYnsha 1728 quick-start\n' +
-          'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB 1102 readme\n' +
-          'QmTumTjvcYCAvRRwQ8sDRxh8ezmrcr88YFU7iYNroGGTBZ 1027 security-notes\n' +
-          'QmciSU8hfpAXKjvK5YLUSwApomGSWN5gFbP4EpDAEzu2Te 863  tour/\n'
+          'QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V 1677 about\n' +
+          'QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y 189  contact\n' +
+          'QmegvLXxpVKiZ4b57Xs1syfBVRd8CbucVHAp7KpLQdGieC -    docs/\n' +
+          'QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 311  help\n' +
+          'QmdncfsVm2h5Kqq9hPmU7oAVX2zTSVP3L869tgTbPYnsha 1717 quick-start\n' +
+          'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB 1091 readme\n' +
+          'QmTumTjvcYCAvRRwQ8sDRxh8ezmrcr88YFU7iYNroGGTBZ 1016 security-notes\n' +
+          'QmciSU8hfpAXKjvK5YLUSwApomGSWN5gFbP4EpDAEzu2Te -    tour/\n'
         )
       })
   })
@@ -76,16 +76,16 @@ describe('ls', () => runOnAndOff((thing) => {
     return ipfs('ls -r /ipfs/Qmaj2NmcyAXT8dFmZRRytE12wpcaHADzbChKToMEjBsj5Z/init-docs')
       .then(out => {
         expect(out).to.eql(
-          'QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V 1688 about\n' +
-          'QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y 200  contact\n' +
-          'QmegvLXxpVKiZ4b57Xs1syfBVRd8CbucVHAp7KpLQdGieC 65   docs/\n' +
-          'QmQN88TEidd3RY2u3dpib49fERTDfKtDpvxnvczATNsfKT 14     index\n' +
-          'QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 322  help\n' +
-          'QmdncfsVm2h5Kqq9hPmU7oAVX2zTSVP3L869tgTbPYnsha 1728 quick-start\n' +
-          'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB 1102 readme\n' +
-          'QmTumTjvcYCAvRRwQ8sDRxh8ezmrcr88YFU7iYNroGGTBZ 1027 security-notes\n' +
-          'QmciSU8hfpAXKjvK5YLUSwApomGSWN5gFbP4EpDAEzu2Te 863  tour/\n' +
-          'QmYE7xo6NxbHEVEHej1yzxijYaNY51BaeKxjXxn6Ssa6Bs 807    0.0-intro\n'
+          'QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V 1677 about\n' +
+          'QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y 189  contact\n' +
+          'QmegvLXxpVKiZ4b57Xs1syfBVRd8CbucVHAp7KpLQdGieC -    docs/\n' +
+          'QmQN88TEidd3RY2u3dpib49fERTDfKtDpvxnvczATNsfKT 6      index\n' +
+          'QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 311  help\n' +
+          'QmdncfsVm2h5Kqq9hPmU7oAVX2zTSVP3L869tgTbPYnsha 1717 quick-start\n' +
+          'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB 1091 readme\n' +
+          'QmTumTjvcYCAvRRwQ8sDRxh8ezmrcr88YFU7iYNroGGTBZ 1016 security-notes\n' +
+          'QmciSU8hfpAXKjvK5YLUSwApomGSWN5gFbP4EpDAEzu2Te -    tour/\n' +
+          'QmYE7xo6NxbHEVEHej1yzxijYaNY51BaeKxjXxn6Ssa6Bs 796    0.0-intro\n'
         )
       })
   })
@@ -96,11 +96,11 @@ describe('ls', () => runOnAndOff((thing) => {
     return ipfs('ls Qmaj2NmcyAXT8dFmZRRytE12wpcaHADzbChKToMEjBsj5Z --cid-base=base64')
       .then((out) => {
         expect(out).to.eql(
-          'mAXASILidvV1YroHLqBvmuXko1Ly1UVenZV1K+MvhsjXhdvZQ 123530 blocks/\n' +
-          'mAXASIBT4ZYkQw0IApLoNHBxSjpezyayKZHJyxmFKpt0I3sK5 3939   config\n' +
-          'mAXASIGCpScP8zpa0CqUgyVCR/Cm0Co8pnULGe3seXSsOnJsJ 5503   datastore/\n' +
-          'mAXASIF58POI3+TbHb69iXpD3dRqfXusEj1mHMwPCFenM6HWZ 7397   init-docs/\n' +
-          'mAXASICiW5ai+KiU60glImEMMkiHaNSOAivpXspriIhJO8iHI 10     version\n'
+          'mAXASILidvV1YroHLqBvmuXko1Ly1UVenZV1K+MvhsjXhdvZQ -    blocks/\n' +
+          'mAXASIBT4ZYkQw0IApLoNHBxSjpezyayKZHJyxmFKpt0I3sK5 3928 config\n' +
+          'mAXASIGCpScP8zpa0CqUgyVCR/Cm0Co8pnULGe3seXSsOnJsJ -    datastore/\n' +
+          'mAXASIF58POI3+TbHb69iXpD3dRqfXusEj1mHMwPCFenM6HWZ -    init-docs/\n' +
+          'mAXASICiW5ai+KiU60glImEMMkiHaNSOAivpXspriIhJO8iHI 2    version\n'
         )
       })
   })

test/core/interface.spec.js

@@ -62,16 +62,7 @@ describe('interface-ipfs-core tests', function () {
   })
 
   tests.filesRegular(defaultCommonFactory, {
-    skip: isNode ? [{
-      name: 'should ls with a base58 encoded CID',
-      reason: 'File sizes incompatible with go-ipfs https://github.com/ipfs/js-ipfs/issues/1934'
-    }, {
-      name: 'should readable stream ls with a base58 encoded CID',
-      reason: 'File sizes incompatible with go-ipfs https://github.com/ipfs/js-ipfs/issues/1934'
-    }, {
-      name: 'should pull stream ls with a base58 encoded CID',
-      reason: 'File sizes incompatible with go-ipfs https://github.com/ipfs/js-ipfs/issues/1934'
-    }] : [{
+    skip: isNode ? null : [{
       name: 'addFromStream',
       reason: 'Not designed to run in the browser'
     }, {
@@ -80,15 +71,6 @@ describe('interface-ipfs-core tests', function () {
     }, {
       name: 'addFromURL',
       reason: 'Not designed to run in the browser'
-    }, {
-      name: 'should ls with a base58 encoded CID',
-      reason: 'File sizes incompatible with go-ipfs https://github.com/ipfs/js-ipfs/issues/1934'
-    }, {
-      name: 'should readable stream ls with a base58 encoded CID',
-      reason: 'File sizes incompatible with go-ipfs https://github.com/ipfs/js-ipfs/issues/1934'
-    }, {
-      name: 'should pull stream ls with a base58 encoded CID',
-      reason: 'File sizes incompatible with go-ipfs https://github.com/ipfs/js-ipfs/issues/1934'
     }]
   })
 

test/http-api/interface.js

@@ -46,18 +46,7 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => {
     ]
   })
 
-  tests.filesRegular(defaultCommonFactory, {
-    skip: [{
-      name: 'should ls with a base58 encoded CID',
-      reason: 'File sizes incompatible with go-ipfs https://github.com/ipfs/js-ipfs/issues/1934'
-    }, {
-      name: 'should readable stream ls with a base58 encoded CID',
-      reason: 'File sizes incompatible with go-ipfs https://github.com/ipfs/js-ipfs/issues/1934'
-    }, {
-      name: 'should pull stream ls with a base58 encoded CID',
-      reason: 'File sizes incompatible with go-ipfs https://github.com/ipfs/js-ipfs/issues/1934'
-    }]
-  })
+  tests.filesRegular(defaultCommonFactory)
 
   tests.filesMFS(defaultCommonFactory)