diff --git a/get.js b/get.js index 2bb3afa..6ea6914 100644 --- a/get.js +++ b/get.js @@ -2,6 +2,7 @@ const BB = require('bluebird') +const figgyPudding = require('figgy-pudding') const fs = require('fs') const index = require('./lib/entry-index') const memo = require('./lib/memoization') @@ -10,6 +11,12 @@ const pipeline = require('mississippi').pipeline const read = require('./lib/content/read') const through = require('mississippi').through +const GetOpts = figgyPudding({ + integrity: {}, + memoize: {}, + size: {} +}) + module.exports = function get (cache, key, opts) { return getData(false, cache, key, opts) } @@ -17,7 +24,7 @@ module.exports.byDigest = function getByDigest (cache, digest, opts) { return getData(true, cache, digest, opts) } function getData (byDigest, cache, key, opts) { - opts = opts || {} + opts = GetOpts(opts) const memoized = ( byDigest ? memo.get.byDigest(cache, key, opts) @@ -58,7 +65,7 @@ function getData (byDigest, cache, key, opts) { module.exports.stream = getStream function getStream (cache, key, opts) { - opts = opts || {} + opts = GetOpts(opts) let stream = through() const memoized = memo.get(cache, key, opts) if (memoized && opts.memoize !== false) { @@ -91,7 +98,6 @@ function getStream (cache, key, opts) { } else { memoStream = through() } - opts.size = opts.size == null ? entry.size : opts.size stream.emit('metadata', entry.metadata) stream.emit('integrity', entry.integrity) stream.emit('size', entry.size) @@ -101,7 +107,9 @@ function getStream (cache, key, opts) { ev === 'size' && cb(entry.size) }) pipe( - read.readStream(cache, entry.integrity, opts), + read.readStream(cache, entry.integrity, opts.concat({ + size: opts.size == null ? entry.size : opts.size + })), memoStream, stream ) @@ -111,7 +119,7 @@ function getStream (cache, key, opts) { module.exports.stream.byDigest = getStreamDigest function getStreamDigest (cache, integrity, opts) { - opts = opts || {} + opts = GetOpts(opts) const memoized = memo.get.byDigest(cache, integrity, opts) if (memoized && opts.memoize !== false) { const stream = through() @@ -143,7 +151,7 @@ function getStreamDigest (cache, integrity, opts) { module.exports.info = info function info (cache, key, opts) { - opts = opts || {} + opts = GetOpts(opts) const memoized = memo.get(cache, key, opts) if (memoized && opts.memoize !== false) { return BB.resolve(memoized.entry) @@ -161,7 +169,7 @@ module.exports.copy.byDigest = function cpDigest (cache, digest, dest, opts) { return copy(true, cache, digest, dest, opts) } function copy (byDigest, cache, key, dest, opts) { - opts = opts || {} + opts = GetOpts(opts) if (read.copy) { return ( byDigest ? BB.resolve(null) : index.find(cache, key, opts) diff --git a/lib/content/read.js b/lib/content/read.js index 7a4da3b..ec99962 100644 --- a/lib/content/read.js +++ b/lib/content/read.js @@ -3,6 +3,7 @@ const BB = require('bluebird') const contentPath = require('./path') +const figgyPudding = require('figgy-pudding') const fs = require('graceful-fs') const PassThrough = require('stream').PassThrough const pipe = BB.promisify(require('mississippi').pipe) @@ -11,9 +12,13 @@ const Y = require('../util/y.js') BB.promisifyAll(fs) +const ReadOpts = figgyPudding({ + size: {} +}) + module.exports = read function read (cache, integrity, opts) { - opts = opts || {} + opts = ReadOpts(opts) return pickContentSri(cache, integrity).then(content => { const sri = content.sri const cpath = contentPath(cache, sri) @@ -32,7 +37,7 @@ function read (cache, integrity, opts) { module.exports.stream = readStream module.exports.readStream = readStream function readStream (cache, integrity, opts) { - opts = opts || {} + opts = ReadOpts(opts) const stream = new PassThrough() pickContentSri( cache, integrity @@ -56,7 +61,7 @@ if (fs.copyFile) { module.exports.copy = copy } function copy (cache, integrity, dest, opts) { - opts = opts || {} + opts = ReadOpts(opts) return pickContentSri(cache, integrity).then(content => { const sri = content.sri const cpath = contentPath(cache, sri) diff --git a/lib/content/write.js b/lib/content/write.js index a79ae92..c713634 100644 --- a/lib/content/write.js +++ b/lib/content/write.js @@ -28,7 +28,9 @@ function write (cache, data, opts) { if (typeof opts.size === 'number' && data.length !== opts.size) { return BB.reject(sizeError(opts.size, data.length)) } - const sri = ssri.fromData(data, opts) + const sri = ssri.fromData(data, { + algorithms: opts.algorithms + }) if (opts.integrity && !ssri.checkData(data, opts.integrity, opts)) { return BB.reject(checksumError(opts.integrity, sri)) } diff --git a/lib/entry-index.js b/lib/entry-index.js index fe1cd06..226f654 100644 --- a/lib/entry-index.js +++ b/lib/entry-index.js @@ -4,6 +4,7 @@ const BB = require('bluebird') const contentPath = require('./content/path') const crypto = require('crypto') +const figgyPudding = require('figgy-pudding') const fixOwner = require('./util/fix-owner') const fs = require('graceful-fs') const hashToSegments = require('./util/hash-to-segments') @@ -29,9 +30,16 @@ module.exports.NotFoundError = class NotFoundError extends Error { } } +const IndexOpts = figgyPudding({ + metadata: {}, + size: {}, + uid: {}, + gid: {} +}) + module.exports.insert = insert function insert (cache, key, integrity, opts) { - opts = opts || {} + opts = IndexOpts(opts) const bucket = bucketPath(cache, key) const entry = { key, diff --git a/lib/util/tmp.js b/lib/util/tmp.js index 4fc4512..65fc4b2 100644 --- a/lib/util/tmp.js +++ b/lib/util/tmp.js @@ -2,14 +2,21 @@ const BB = require('bluebird') +const figgyPudding = require('figgy-pudding') const fixOwner = require('./fix-owner') const path = require('path') const rimraf = BB.promisify(require('rimraf')) const uniqueFilename = require('unique-filename') +const TmpOpts = figgyPudding({ + tmpPrefix: {}, + uid: {}, + gid: {} +}) + module.exports.mkdir = mktmpdir function mktmpdir (cache, opts) { - opts = opts || {} + opts = TmpOpts(opts) const tmpTarget = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix) return fixOwner.mkdirfix(tmpTarget, opts.uid, opts.gid).then(() => { return tmpTarget @@ -22,11 +29,12 @@ function withTmp (cache, opts, cb) { cb = opts opts = null } - opts = opts || {} + opts = TmpOpts(opts) return BB.using(mktmpdir(cache, opts).disposer(rimraf), cb) } module.exports.fix = fixtmpdir function fixtmpdir (cache, opts) { + opts = TmpOpts(opts) return fixOwner(path.join(cache, 'tmp'), opts.uid, opts.gid) } diff --git a/lib/verify.js b/lib/verify.js index 6a01004..80d59bd 100644 --- a/lib/verify.js +++ b/lib/verify.js @@ -3,6 +3,7 @@ const BB = require('bluebird') const contentPath = require('./content/path') +const figgyPudding = require('figgy-pudding') const finished = BB.promisify(require('mississippi').finished) const fixOwner = require('./util/fix-owner') const fs = require('graceful-fs') @@ -14,10 +15,22 @@ const ssri = require('ssri') BB.promisifyAll(fs) +const VerifyOpts = figgyPudding({ + concurrency: { + default: 20 + }, + filter: {}, + log: { + default: { silly () {} } + }, + uid: {}, + gid: {} +}) + module.exports = verify function verify (cache, opts) { - opts = opts || {} - opts.log && opts.log.silly('verify', 'verifying cache at', cache) + opts = VerifyOpts(opts) + opts.log.silly('verify', 'verifying cache at', cache) return BB.reduce([ markStartTime, fixPerms, @@ -40,7 +53,7 @@ function verify (cache, opts) { }) }, {}).tap(stats => { stats.runTime.total = stats.endTime - stats.startTime - opts.log && opts.log.silly('verify', 'verification finished for', cache, 'in', `${stats.runTime.total}ms`) + opts.log.silly('verify', 'verification finished for', cache, 'in', `${stats.runTime.total}ms`) }) } @@ -53,7 +66,7 @@ function markEndTime (cache, opts) { } function fixPerms (cache, opts) { - opts.log && opts.log.silly('verify', 'fixing cache permissions') + opts.log.silly('verify', 'fixing cache permissions') return fixOwner.mkdirfix(cache, opts.uid, opts.gid).then(() => { // TODO - fix file permissions too return fixOwner.chownr(cache, opts.uid, opts.gid) @@ -70,11 +83,11 @@ function fixPerms (cache, opts) { // 5. If content is not marked as live, rimraf it. // function garbageCollect (cache, opts) { - opts.log && opts.log.silly('verify', 'garbage collecting content') + opts.log.silly('verify', 'garbage collecting content') const indexStream = index.lsStream(cache) const liveContent = new Set() indexStream.on('data', entry => { - if (opts && opts.filter && !opts.filter(entry)) { return } + if (opts.filter && !opts.filter(entry)) { return } liveContent.add(entry.integrity.toString()) }) return finished(indexStream).then(() => { @@ -117,7 +130,7 @@ function garbageCollect (cache, opts) { }) }) } - }, {concurrency: opts.concurrency || 20})) + }, {concurrency: opts.concurrency})) }) }) } @@ -141,7 +154,7 @@ function verifyContent (filepath, sri) { } function rebuildIndex (cache, opts) { - opts.log && opts.log.silly('verify', 'rebuilding index') + opts.log.silly('verify', 'rebuilding index') return index.ls(cache).then(entries => { const stats = { missingContent: 0, @@ -153,7 +166,7 @@ function rebuildIndex (cache, opts) { if (entries.hasOwnProperty(k)) { const hashed = index._hashKey(k) const entry = entries[k] - const excluded = opts && opts.filter && !opts.filter(entry) + const excluded = opts.filter && !opts.filter(entry) excluded && stats.rejectedEntries++ if (buckets[hashed] && !excluded) { buckets[hashed].push(entry) @@ -170,7 +183,7 @@ function rebuildIndex (cache, opts) { } return BB.map(Object.keys(buckets), key => { return rebuildBucket(cache, buckets[key], stats, opts) - }, {concurrency: opts.concurrency || 20}).then(() => stats) + }, {concurrency: opts.concurrency}).then(() => stats) }) } @@ -195,13 +208,13 @@ function rebuildBucket (cache, bucket, stats, opts) { } function cleanTmp (cache, opts) { - opts.log && opts.log.silly('verify', 'cleaning tmp directory') + opts.log.silly('verify', 'cleaning tmp directory') return rimraf(path.join(cache, 'tmp')) } function writeVerifile (cache, opts) { const verifile = path.join(cache, '_lastverified') - opts.log && opts.log.silly('verify', 'writing verifile to ' + verifile) + opts.log.silly('verify', 'writing verifile to ' + verifile) return fs.writeFileAsync(verifile, '' + (+(new Date()))) } diff --git a/package-lock.json b/package-lock.json index a655ab5..e86e173 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1375,6 +1375,11 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "figgy-pudding": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.1.0.tgz", + "integrity": "sha512-Gi2vIue0ec6P/7LNpueGhLuvfF2ztuterl8YFBQn1yKgIS46noGxCbi+vviPdObNYtgUSh5FpHy5q0Cw9XhxKQ==" + }, "figures": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/figures/-/figures-1.7.0.tgz", diff --git a/package.json b/package.json index f3dda96..876dda2 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "dependencies": { "bluebird": "^3.5.1", "chownr": "^1.0.1", + "figgy-pudding": "^3.1.0", "glob": "^7.1.2", "graceful-fs": "^4.1.11", "lru-cache": "^4.1.1", diff --git a/put.js b/put.js index fe1293e..0b0ee14 100644 --- a/put.js +++ b/put.js @@ -1,17 +1,31 @@ 'use strict' +const figgyPudding = require('figgy-pudding') const index = require('./lib/entry-index') const memo = require('./lib/memoization') const write = require('./lib/content/write') const to = require('mississippi').to +const PutOpts = figgyPudding({ + algorithms: { + default: ['sha512'] + }, + integrity: {}, + memoize: {}, + metadata: {}, + size: {}, + tmpPrefix: {}, + uid: {}, + gid: {} +}) + module.exports = putData function putData (cache, key, data, opts) { - opts = opts || {} + opts = PutOpts(opts) return write(cache, data, opts).then(res => { - // TODO - stop modifying opts - opts.size = res.size - return index.insert(cache, key, res.integrity, opts).then(entry => { + return index.insert( + cache, key, res.integrity, opts.concat({size: res.size}) + ).then(entry => { if (opts.memoize) { memo.put(cache, entry, data, opts) } @@ -22,7 +36,7 @@ function putData (cache, key, data, opts) { module.exports.stream = putStream function putStream (cache, key, opts) { - opts = opts || {} + opts = PutOpts(opts) let integrity let size const contentStream = write.stream( @@ -45,9 +59,7 @@ function putStream (cache, key, opts) { }) }, cb => { contentStream.end(() => { - // TODO - stop modifying `opts` - opts.size = size - index.insert(cache, key, integrity, opts).then(entry => { + index.insert(cache, key, integrity, opts.concat({size})).then(entry => { if (opts.memoize) { memo.put(cache, entry, Buffer.concat(memoData, memoTotal), opts) }