Skip to content
This repository has been archived by the owner on Jul 3, 2019. It is now read-only.

feat(opts): use figgy-pudding for opts #128

Merged
merged 1 commit into from
Apr 9, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions get.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

const BB = require('bluebird')

const figgyPudding = require('figgy-pudding')
const fs = require('fs')
const index = require('./lib/entry-index')
const memo = require('./lib/memoization')
Expand All @@ -10,14 +11,20 @@ const pipeline = require('mississippi').pipeline
const read = require('./lib/content/read')
const through = require('mississippi').through

const GetOpts = figgyPudding({
integrity: {},
memoize: {},
size: {}
})

module.exports = function get (cache, key, opts) {
return getData(false, cache, key, opts)
}
module.exports.byDigest = function getByDigest (cache, digest, opts) {
return getData(true, cache, digest, opts)
}
function getData (byDigest, cache, key, opts) {
opts = opts || {}
opts = GetOpts(opts)
const memoized = (
byDigest
? memo.get.byDigest(cache, key, opts)
Expand Down Expand Up @@ -58,7 +65,7 @@ function getData (byDigest, cache, key, opts) {

module.exports.stream = getStream
function getStream (cache, key, opts) {
opts = opts || {}
opts = GetOpts(opts)
let stream = through()
const memoized = memo.get(cache, key, opts)
if (memoized && opts.memoize !== false) {
Expand Down Expand Up @@ -91,7 +98,6 @@ function getStream (cache, key, opts) {
} else {
memoStream = through()
}
opts.size = opts.size == null ? entry.size : opts.size
stream.emit('metadata', entry.metadata)
stream.emit('integrity', entry.integrity)
stream.emit('size', entry.size)
Expand All @@ -101,7 +107,9 @@ function getStream (cache, key, opts) {
ev === 'size' && cb(entry.size)
})
pipe(
read.readStream(cache, entry.integrity, opts),
read.readStream(cache, entry.integrity, opts.concat({
size: opts.size == null ? entry.size : opts.size
})),
memoStream,
stream
)
Expand All @@ -111,7 +119,7 @@ function getStream (cache, key, opts) {

module.exports.stream.byDigest = getStreamDigest
function getStreamDigest (cache, integrity, opts) {
opts = opts || {}
opts = GetOpts(opts)
const memoized = memo.get.byDigest(cache, integrity, opts)
if (memoized && opts.memoize !== false) {
const stream = through()
Expand Down Expand Up @@ -143,7 +151,7 @@ function getStreamDigest (cache, integrity, opts) {

module.exports.info = info
function info (cache, key, opts) {
opts = opts || {}
opts = GetOpts(opts)
const memoized = memo.get(cache, key, opts)
if (memoized && opts.memoize !== false) {
return BB.resolve(memoized.entry)
Expand All @@ -161,7 +169,7 @@ module.exports.copy.byDigest = function cpDigest (cache, digest, dest, opts) {
return copy(true, cache, digest, dest, opts)
}
function copy (byDigest, cache, key, dest, opts) {
opts = opts || {}
opts = GetOpts(opts)
if (read.copy) {
return (
byDigest ? BB.resolve(null) : index.find(cache, key, opts)
Expand Down
11 changes: 8 additions & 3 deletions lib/content/read.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
const BB = require('bluebird')

const contentPath = require('./path')
const figgyPudding = require('figgy-pudding')
const fs = require('graceful-fs')
const PassThrough = require('stream').PassThrough
const pipe = BB.promisify(require('mississippi').pipe)
Expand All @@ -11,9 +12,13 @@ const Y = require('../util/y.js')

BB.promisifyAll(fs)

const ReadOpts = figgyPudding({
size: {}
})

module.exports = read
function read (cache, integrity, opts) {
opts = opts || {}
opts = ReadOpts(opts)
return pickContentSri(cache, integrity).then(content => {
const sri = content.sri
const cpath = contentPath(cache, sri)
Expand All @@ -32,7 +37,7 @@ function read (cache, integrity, opts) {
module.exports.stream = readStream
module.exports.readStream = readStream
function readStream (cache, integrity, opts) {
opts = opts || {}
opts = ReadOpts(opts)
const stream = new PassThrough()
pickContentSri(
cache, integrity
Expand All @@ -56,7 +61,7 @@ if (fs.copyFile) {
module.exports.copy = copy
}
function copy (cache, integrity, dest, opts) {
opts = opts || {}
opts = ReadOpts(opts)
return pickContentSri(cache, integrity).then(content => {
const sri = content.sri
const cpath = contentPath(cache, sri)
Expand Down
4 changes: 3 additions & 1 deletion lib/content/write.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ function write (cache, data, opts) {
if (typeof opts.size === 'number' && data.length !== opts.size) {
return BB.reject(sizeError(opts.size, data.length))
}
const sri = ssri.fromData(data, opts)
const sri = ssri.fromData(data, {
algorithms: opts.algorithms
})
if (opts.integrity && !ssri.checkData(data, opts.integrity, opts)) {
return BB.reject(checksumError(opts.integrity, sri))
}
Expand Down
10 changes: 9 additions & 1 deletion lib/entry-index.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ const BB = require('bluebird')

const contentPath = require('./content/path')
const crypto = require('crypto')
const figgyPudding = require('figgy-pudding')
const fixOwner = require('./util/fix-owner')
const fs = require('graceful-fs')
const hashToSegments = require('./util/hash-to-segments')
Expand All @@ -29,9 +30,16 @@ module.exports.NotFoundError = class NotFoundError extends Error {
}
}

const IndexOpts = figgyPudding({
metadata: {},
size: {},
uid: {},
gid: {}
})

module.exports.insert = insert
function insert (cache, key, integrity, opts) {
opts = opts || {}
opts = IndexOpts(opts)
const bucket = bucketPath(cache, key)
const entry = {
key,
Expand Down
12 changes: 10 additions & 2 deletions lib/util/tmp.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,21 @@

const BB = require('bluebird')

const figgyPudding = require('figgy-pudding')
const fixOwner = require('./fix-owner')
const path = require('path')
const rimraf = BB.promisify(require('rimraf'))
const uniqueFilename = require('unique-filename')

const TmpOpts = figgyPudding({
tmpPrefix: {},
uid: {},
gid: {}
})

module.exports.mkdir = mktmpdir
function mktmpdir (cache, opts) {
opts = opts || {}
opts = TmpOpts(opts)
const tmpTarget = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
return fixOwner.mkdirfix(tmpTarget, opts.uid, opts.gid).then(() => {
return tmpTarget
Expand All @@ -22,11 +29,12 @@ function withTmp (cache, opts, cb) {
cb = opts
opts = null
}
opts = opts || {}
opts = TmpOpts(opts)
return BB.using(mktmpdir(cache, opts).disposer(rimraf), cb)
}

module.exports.fix = fixtmpdir
function fixtmpdir (cache, opts) {
opts = TmpOpts(opts)
return fixOwner(path.join(cache, 'tmp'), opts.uid, opts.gid)
}
37 changes: 25 additions & 12 deletions lib/verify.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
const BB = require('bluebird')

const contentPath = require('./content/path')
const figgyPudding = require('figgy-pudding')
const finished = BB.promisify(require('mississippi').finished)
const fixOwner = require('./util/fix-owner')
const fs = require('graceful-fs')
Expand All @@ -14,10 +15,22 @@ const ssri = require('ssri')

BB.promisifyAll(fs)

const VerifyOpts = figgyPudding({
concurrency: {
default: 20
},
filter: {},
log: {
default: { silly () {} }
},
uid: {},
gid: {}
})

module.exports = verify
function verify (cache, opts) {
opts = opts || {}
opts.log && opts.log.silly('verify', 'verifying cache at', cache)
opts = VerifyOpts(opts)
opts.log.silly('verify', 'verifying cache at', cache)
return BB.reduce([
markStartTime,
fixPerms,
Expand All @@ -40,7 +53,7 @@ function verify (cache, opts) {
})
}, {}).tap(stats => {
stats.runTime.total = stats.endTime - stats.startTime
opts.log && opts.log.silly('verify', 'verification finished for', cache, 'in', `${stats.runTime.total}ms`)
opts.log.silly('verify', 'verification finished for', cache, 'in', `${stats.runTime.total}ms`)
})
}

Expand All @@ -53,7 +66,7 @@ function markEndTime (cache, opts) {
}

function fixPerms (cache, opts) {
opts.log && opts.log.silly('verify', 'fixing cache permissions')
opts.log.silly('verify', 'fixing cache permissions')
return fixOwner.mkdirfix(cache, opts.uid, opts.gid).then(() => {
// TODO - fix file permissions too
return fixOwner.chownr(cache, opts.uid, opts.gid)
Expand All @@ -70,11 +83,11 @@ function fixPerms (cache, opts) {
// 5. If content is not marked as live, rimraf it.
//
function garbageCollect (cache, opts) {
opts.log && opts.log.silly('verify', 'garbage collecting content')
opts.log.silly('verify', 'garbage collecting content')
const indexStream = index.lsStream(cache)
const liveContent = new Set()
indexStream.on('data', entry => {
if (opts && opts.filter && !opts.filter(entry)) { return }
if (opts.filter && !opts.filter(entry)) { return }
liveContent.add(entry.integrity.toString())
})
return finished(indexStream).then(() => {
Expand Down Expand Up @@ -117,7 +130,7 @@ function garbageCollect (cache, opts) {
})
})
}
}, {concurrency: opts.concurrency || 20}))
}, {concurrency: opts.concurrency}))
})
})
}
Expand All @@ -141,7 +154,7 @@ function verifyContent (filepath, sri) {
}

function rebuildIndex (cache, opts) {
opts.log && opts.log.silly('verify', 'rebuilding index')
opts.log.silly('verify', 'rebuilding index')
return index.ls(cache).then(entries => {
const stats = {
missingContent: 0,
Expand All @@ -153,7 +166,7 @@ function rebuildIndex (cache, opts) {
if (entries.hasOwnProperty(k)) {
const hashed = index._hashKey(k)
const entry = entries[k]
const excluded = opts && opts.filter && !opts.filter(entry)
const excluded = opts.filter && !opts.filter(entry)
excluded && stats.rejectedEntries++
if (buckets[hashed] && !excluded) {
buckets[hashed].push(entry)
Expand All @@ -170,7 +183,7 @@ function rebuildIndex (cache, opts) {
}
return BB.map(Object.keys(buckets), key => {
return rebuildBucket(cache, buckets[key], stats, opts)
}, {concurrency: opts.concurrency || 20}).then(() => stats)
}, {concurrency: opts.concurrency}).then(() => stats)
})
}

Expand All @@ -195,13 +208,13 @@ function rebuildBucket (cache, bucket, stats, opts) {
}

function cleanTmp (cache, opts) {
opts.log && opts.log.silly('verify', 'cleaning tmp directory')
opts.log.silly('verify', 'cleaning tmp directory')
return rimraf(path.join(cache, 'tmp'))
}

function writeVerifile (cache, opts) {
const verifile = path.join(cache, '_lastverified')
opts.log && opts.log.silly('verify', 'writing verifile to ' + verifile)
opts.log.silly('verify', 'writing verifile to ' + verifile)
return fs.writeFileAsync(verifile, '' + (+(new Date())))
}

Expand Down
Loading