Skip to content

Commit

Permalink
check for cached packages through packument in addition to url parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@fritzy
fritzy committed Aug 18, 2021
1 parent 5bd6805 commit 2adabd1
Show file tree
Hide file tree
Showing 2 changed files with 128 additions and 29 deletions.
57 changes: 45 additions & 12 deletions lib/cache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,56 @@ const rimraf = promisify(require('rimraf'))
const semver = require('semver')
const BaseCommand = require('./base-command.js')
const npa = require('npm-package-arg')
const jsonParse = require('json-parse-even-better-errors')

const searchCachePackage = (spec, cacheKeys) => {
const searchCachePackage = async (path, spec, cacheKeys) => {
const parsed = npa(spec)
if (parsed.rawSpec !== '' && parsed.type === 'tag')
throw new Error(`Cannot list cache keys for a tagged package.`)
const searchMFH = new RegExp(`^make-fetch-happen:request-cache:.*(?<!/[@a-zA-Z]+)/${parsed.name}/-/([^/]+.tgz)$`)
const searchMFH = new RegExp(`^make-fetch-happen:request-cache:.*(?<!/[@a-zA-Z]+)/${parsed.name}/-/(${parsed.name}[^/]+.tgz)$`)
const searchPack = new RegExp(`^make-fetch-happen:request-cache:.*/${parsed.escapedName}$`)
const results = cacheKeys.filter(key => {
const results = new Set()
cacheKeys = new Set(cacheKeys)
for (const key of cacheKeys) {
// match on the public key registry url format
if (searchMFH.test(key)) {
// extract the version from the filename
const filename = key.match(searchMFH)[1]
const noExt = filename.slice(0, -4)
const noScope = `${parsed.name.split('/').pop()}-`
const ver = noExt.slice(noScope.length)
return semver.satisfies(ver, parsed.rawSpec)
if (semver.satisfies(ver, parsed.rawSpec))
results.add(key)
continue
}
return searchPack.test(key)
})
// is this key a packument?
if (!searchPack.test(key))
continue

results.add(key)
let packument, details
try {
details = await cacache.get(path, key)
packument = jsonParse(details.data)
} catch (_) {
// if we couldn't parse the packument, abort
/* istanbul ignore next */
continue
}
/* istanbul ignore next */
if (!packument.versions || typeof packument.versions !== 'object')
continue
// assuming this is a packument
for (const ver of Object.keys(packument.versions)) {
if (semver.satisfies(ver, parsed.rawSpec)) {
/* istanbul ignore else */
if (packument.versions[ver].dist
&& typeof packument.versions[ver].dist === 'object'
&& cacheKeys.has(`make-fetch-happen:request-cache:${packument.versions[ver].dist.tarball}`))
results.add(`make-fetch-happen:request-cache:${packument.versions[ver].dist.tarball}`)
}
}
}
return results
}

Expand Down Expand Up @@ -177,12 +209,13 @@ class Cache extends BaseCommand {
if (Array.isArray(this.npm.flatOptions.package)
&& this.npm.flatOptions.package.length > 0) {
// get results for each package spec specified
const keyLists = this.npm.flatOptions.package.map(
spec => searchCachePackage(spec, cacheKeys)
)
// flatten list and output each key
keyLists.reduce((keyList, keyArr) => keyList.concat(keyArr))
.forEach((key) => this.npm.output(key))
const results = new Set()
for (const spec of this.npm.flatOptions.package) {
const keySet = await searchCachePackage(cachePath, spec, cacheKeys)
for (const key of keySet)
results.add(key)
}
[...results].sort().forEach(key => this.npm.output(key))
return
}
cacheKeys.sort().forEach(key => this.npm.output(key))
Expand Down
100 changes: 83 additions & 17 deletions test/lib/cache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,21 @@ const setupCacacheFixture = () => {
cacacheEntries = {}
cacacheContent = {}
const pkgs = [
'webpack@4.44.1',
'npm@1.2.0',
'webpack@4.47.0',
'foo@1.2.3-beta',
['webpack@4.44.1', 'https://registry.npmjs.org', true],
['npm@1.2.0', 'https://registry.npmjs.org', true],
['webpack@4.47.0', 'https://registry.npmjs.org', true],
['foo@1.2.3-beta', 'https://registry.npmjs.org', true],
['ape-ecs@2.1.7', 'https://registry.npmjs.org', true],
['@fritzy/staydown@3.1.1', 'https://registry.npmjs.org', true],
['@gar/npm-expansion@2.1.0', 'https://registry.npmjs.org', true],
['@gar/npm-expansion@3.0.0-beta', 'https://registry.npmjs.org', true],
['extemporaneously@44.2.2', 'https://somerepo.github.org', false],
]
pkgs.forEach(addCacachePkg)
pkgs.forEach(pkg => addCacachePkg(...pkg))
}

const packuments = {}

let contentId = 0
const cacacheVerifyStats = {
keptSize: 100,
Expand All @@ -58,27 +65,45 @@ const cacacheVerifyStats = {
runTime: { total: 2000 },
}

const addCacacheKey = (key) => {
const addCacacheKey = (key, content) => {
contentId++
cacacheEntries[key] = { integrity: `${contentId}` }
cacacheContent[`${contentId}`] = {}
}
const addCacachePkg = (spec) => {
const addCacachePkg = (spec, registry, publicURL) => {
const parts = npa(spec)
const key = `make-fetch-happen:request-cache:https://registry.npmjs.org/${parts.name}/-/${parts.name}-${parts.rawSpec || '1.0.0'}.tgz`
const ver = parts.rawSpec || '1.0.0'
let url = `${registry}/${parts.name}/-/${parts.name}-${ver}.tgz`
if (!publicURL)
url = `${registry}/aabbcc/${contentId}`
const key = `make-fetch-happen:request-cache:${url}`
const pkey = `make-fetch-happen:request-cache:${registry}/${parts.escapedName}`
if (!packuments[parts.escapedName]) {
packuments[parts.escapedName] = {
versions: {},
}
addCacacheKey(pkey)
}
packuments[parts.escapedName].versions[ver] = {
dist: {
tarball: url,
},
}
addCacacheKey(key)
const key2 = `make-fetch-happen:request-cache:https://registry.npmjs.org/${parts.escapedName}`
addCacacheKey(key2)
cacacheContent[cacacheEntries[pkey].integrity] = {
data: Buffer.from(JSON.stringify(packuments[parts.escapedName])),
}
}

const cacache = {
verify: (path) => {
return cacacheVerifyStats
},
get: (path, key) => {
if (cacacheEntries[key] === undefined)
if (cacacheEntries[key] === undefined
|| cacacheContent[cacacheEntries[key].integrity] === undefined)
throw new Error()
return cacacheEntries[key]
return cacacheContent[cacacheEntries[key].integrity]
},
rm: {
entry: (path, key) => {
Expand All @@ -87,8 +112,6 @@ const cacache = {
delete cacacheEntries[key]
},
content: (path, sha) => {
if (cacacheContent[sha] === undefined)
throw new Error()
delete cacacheContent[sha]
},
},
Expand Down Expand Up @@ -213,13 +236,22 @@ t.test('cache ls', t => {
cache.exec(['ls'], err => {
t.error(err)
t.strictSame(outputOutput, [
'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy%2fstaydown',
'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy/staydown/-/@fritzy/staydown-3.1.1.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar%2fnpm-expansion',
'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar/npm-expansion/-/@gar/npm-expansion-2.1.0.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar/npm-expansion/-/@gar/npm-expansion-3.0.0-beta.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/ape-ecs',
'make-fetch-happen:request-cache:https://registry.npmjs.org/ape-ecs/-/ape-ecs-2.1.7.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/foo',
'make-fetch-happen:request-cache:https://registry.npmjs.org/foo/-/foo-1.2.3-beta.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/npm',
'make-fetch-happen:request-cache:https://registry.npmjs.org/npm/-/npm-1.2.0.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack',
'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.44.1.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.47.0.tgz',
'make-fetch-happen:request-cache:https://somerepo.github.org/aabbcc/14',
'make-fetch-happen:request-cache:https://somerepo.github.org/extemporaneously',
])
t.end()
})
Expand All @@ -234,10 +266,10 @@ t.test('cache ls pkgs', t => {
cache.exec(['ls'], err => {
t.error(err)
t.strictSame(outputOutput, [
'make-fetch-happen:request-cache:https://registry.npmjs.org/npm',
'make-fetch-happen:request-cache:https://registry.npmjs.org/npm/-/npm-1.2.0.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack',
'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.47.0.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/npm/-/npm-1.2.0.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/npm',
])
t.end()
})
Expand All @@ -252,8 +284,24 @@ t.test('cache ls special', t => {
cache.exec(['ls'], err => {
t.error(err)
t.strictSame(outputOutput, [
'make-fetch-happen:request-cache:https://registry.npmjs.org/foo/-/foo-1.2.3-beta.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/foo',
'make-fetch-happen:request-cache:https://registry.npmjs.org/foo/-/foo-1.2.3-beta.tgz',
])
t.end()
})
})

t.test('cache ls nonpublic registry', t => {
t.teardown(() => {
outputOutput = []
npm.flatOptions.package = undefined
})
npm.flatOptions.package = ['extemporaneously']
cache.exec(['ls'], err => {
t.error(err)
t.strictSame(outputOutput, [
'make-fetch-happen:request-cache:https://somerepo.github.org/aabbcc/14',
'make-fetch-happen:request-cache:https://somerepo.github.org/extemporaneously',
])
t.end()
})
Expand All @@ -271,6 +319,24 @@ t.test('cache ls tagged', t => {
})
})

t.test('cache ls scoped and scoped slash', t => {
t.teardown(() => {
outputOutput = []
npm.flatOptions.package = undefined
})
npm.flatOptions.package = ['@fritzy/staydown', '@gar/npm-expansion']
cache.exec(['ls'], err => {
t.error(err)
t.strictSame(outputOutput, [
'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy%2fstaydown',
'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy/staydown/-/@fritzy/staydown-3.1.1.tgz',
'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar%2fnpm-expansion',
'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar/npm-expansion/-/@gar/npm-expansion-2.1.0.tgz',
])
t.end()
})
})

t.test('cache rm', t => {
t.teardown(() => {
outputOutput = []
Expand Down

0 comments on commit 2adabd1

Please sign in to comment.