diff --git a/lib/cache.js b/lib/cache.js index 55fb3e863631c..aed2cce31e63f 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -4,7 +4,59 @@ const log = require('npmlog') const pacote = require('pacote') const path = require('path') const rimraf = promisify(require('rimraf')) +const semver = require('semver') const BaseCommand = require('./base-command.js') +const npa = require('npm-package-arg') +const jsonParse = require('json-parse-even-better-errors') + +const searchCachePackage = async (path, spec, cacheKeys) => { + const parsed = npa(spec) + if (parsed.rawSpec !== '' && parsed.type === 'tag') + throw new Error(`Cannot list cache keys for a tagged package.`) + const searchMFH = new RegExp(`^make-fetch-happen:request-cache:.*(?', 'add ', 'add @', - 'clean', + 'clean []', + 'ls [@]', 'verify', ] } @@ -37,13 +90,15 @@ class Cache extends BaseCommand { async completion (opts) { const argv = opts.conf.argv.remain if (argv.length === 2) - return ['add', 'clean', 'verify'] + return ['add', 'clean', 'verify', 'ls', 'delete'] // TODO - eventually... switch (argv[2]) { case 'verify': case 'clean': case 'add': + case 'ls': + case 'delete': return [] } } @@ -61,6 +116,8 @@ class Cache extends BaseCommand { return await this.add(args) case 'verify': case 'check': return await this.verify() + case 'ls': + return await this.ls(args) default: throw Object.assign(new Error(this.usage), { code: 'EUSAGE' }) } @@ -68,27 +125,38 @@ class Cache extends BaseCommand { // npm cache clean [pkg]* async clean (args) { - if (args.length) - throw new Error('npm cache clear does not accept arguments') - const cachePath = path.join(this.npm.cache, '_cacache') - if (!this.npm.config.get('force')) { - throw new Error(`As of npm@5, the npm cache self-heals from corruption issues -by treating integrity mismatches as cache misses. As a result, -data extracted from the cache is guaranteed to be valid. If you -want to make sure everything is consistent, use \`npm cache verify\` -instead. Deleting the cache can only make npm go slower, and is -not likely to correct any problems you may be encountering! - -On the other hand, if you're debugging an issue with the installer, -or race conditions that depend on the timing of writing to an empty -cache, you can use \`npm install --cache /tmp/empty-cache\` to use a -temporary cache instead of nuking the actual one. - -If you're sure you want to delete the entire cache, rerun this command -with --force.`) + if (args.length === 0) { + if (!this.npm.config.get('force')) { + throw new Error(`As of npm@5, the npm cache self-heals from corruption issues + by treating integrity mismatches as cache misses. As a result, + data extracted from the cache is guaranteed to be valid. If you + want to make sure everything is consistent, use \`npm cache verify\` + instead. Deleting the cache can only make npm go slower, and is + not likely to correct any problems you may be encountering! + + On the other hand, if you're debugging an issue with the installer, + or race conditions that depend on the timing of writing to an empty + cache, you can use \`npm install --cache /tmp/empty-cache\` to use a + temporary cache instead of nuking the actual one. + + If you're sure you want to delete the entire cache, rerun this command + with --force.`) + } + return rimraf(cachePath) + } + for (const key of args) { + let entry + try { + entry = await cacache.get(cachePath, key) + } catch (err) { + this.npm.log.warn(`Not Found: ${key}`) + break + } + this.npm.output(`Deleted: ${key}`) + await cacache.rm.entry(cachePath, key) + await cacache.rm.content(cachePath, entry.integrity) } - return rimraf(cachePath) } // npm cache add ... @@ -131,6 +199,24 @@ with --force.`) this.npm.output(`Index entries: ${stats.totalEntries}`) this.npm.output(`Finished in ${stats.runTime.total / 1000}s`) } + + // npm cache ls [--package ...] + async ls (specs) { + const cachePath = path.join(this.npm.cache, '_cacache') + const cacheKeys = Object.keys(await cacache.ls(cachePath)) + if (specs.length > 0) { + // get results for each package spec specified + const results = new Set() + for (const spec of specs) { + const keySet = await searchCachePackage(cachePath, spec, cacheKeys) + for (const key of keySet) + results.add(key) + } + [...results].sort((a, b) => a.localeCompare(b, 'en')).forEach(key => this.npm.output(key)) + return + } + cacheKeys.sort((a, b) => a.localeCompare(b, 'en')).forEach(key => this.npm.output(key)) + } } module.exports = Cache diff --git a/tap-snapshots/test/lib/load-all-commands.js.test.cjs b/tap-snapshots/test/lib/load-all-commands.js.test.cjs index 8cf2e2837e295..9f811a0058fd1 100644 --- a/tap-snapshots/test/lib/load-all-commands.js.test.cjs +++ b/tap-snapshots/test/lib/load-all-commands.js.test.cjs @@ -102,7 +102,8 @@ npm cache add npm cache add npm cache add npm cache add @ -npm cache clean +npm cache clean [] +npm cache ls [@] npm cache verify Options: diff --git a/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs b/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs index 50f6481f6e848..0fd36c7c1d372 100644 --- a/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs +++ b/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs @@ -251,7 +251,8 @@ All commands: npm cache add npm cache add npm cache add @ - npm cache clean + npm cache clean [] + npm cache ls [@] npm cache verify Options: diff --git a/test/lib/cache.js b/test/lib/cache.js index d3d6f5b8845de..c6405303202b8 100644 --- a/test/lib/cache.js +++ b/test/lib/cache.js @@ -1,6 +1,7 @@ const t = require('tap') const { fake: mockNpm } = require('../fixtures/mock-npm.js') const path = require('path') +const npa = require('npm-package-arg') const usageUtil = () => 'usage instructions' @@ -34,16 +35,104 @@ const pacote = { }, } +let cacacheEntries = {} +let cacacheContent = {} + +const setupCacacheFixture = () => { + cacacheEntries = {} + cacacheContent = {} + const pkgs = [ + ['webpack@4.44.1', 'https://registry.npmjs.org', true], + ['npm@1.2.0', 'https://registry.npmjs.org', true], + ['webpack@4.47.0', 'https://registry.npmjs.org', true], + ['foo@1.2.3-beta', 'https://registry.npmjs.org', true], + ['ape-ecs@2.1.7', 'https://registry.npmjs.org', true], + ['@fritzy/staydown@3.1.1', 'https://registry.npmjs.org', true], + ['@gar/npm-expansion@2.1.0', 'https://registry.npmjs.org', true], + ['@gar/npm-expansion@3.0.0-beta', 'https://registry.npmjs.org', true], + ['extemporaneously@44.2.2', 'https://somerepo.github.org', false], + ['corrupted@3.1.0', 'https://registry.npmjs.org', true], + ['missing-dist@23.0.0', 'https://registry.npmjs.org', true], + ['missing-version@16.2.0', 'https://registry.npmjs.org', true], + ] + pkgs.forEach(pkg => addCacachePkg(...pkg)) + // corrupt the packument + cacacheContent[ + [cacacheEntries['make-fetch-happen:request-cache:https://registry.npmjs.org/corrupted'].integrity] + ].data = Buffer.from('<>>>}"') + // nuke the version dist + cacacheContent[ + [cacacheEntries['make-fetch-happen:request-cache:https://registry.npmjs.org/missing-dist'].integrity] + ].data = Buffer.from(JSON.stringify({ versions: { '23.0.0': {} } })) + // make the version a non-object + cacacheContent[ + [cacacheEntries['make-fetch-happen:request-cache:https://registry.npmjs.org/missing-version'].integrity] + ].data = Buffer.from(JSON.stringify({ versions: 'hello' })) +} + +const packuments = {} + +let contentId = 0 const cacacheVerifyStats = { keptSize: 100, verifiedContent: 1, totalEntries: 1, runTime: { total: 2000 }, } + +const addCacacheKey = (key, content) => { + contentId++ + cacacheEntries[key] = { integrity: `${contentId}` } + cacacheContent[`${contentId}`] = {} +} +const addCacachePkg = (spec, registry, publicURL) => { + const parts = npa(spec) + const ver = parts.rawSpec || '1.0.0' + let url = `${registry}/${parts.name}/-/${parts.name}-${ver}.tgz` + if (!publicURL) + url = `${registry}/aabbcc/${contentId}` + const key = `make-fetch-happen:request-cache:${url}` + const pkey = `make-fetch-happen:request-cache:${registry}/${parts.escapedName}` + if (!packuments[parts.escapedName]) { + packuments[parts.escapedName] = { + versions: {}, + } + addCacacheKey(pkey) + } + packuments[parts.escapedName].versions[ver] = { + dist: { + tarball: url, + }, + } + addCacacheKey(key) + cacacheContent[cacacheEntries[pkey].integrity] = { + data: Buffer.from(JSON.stringify(packuments[parts.escapedName])), + } +} + const cacache = { verify: (path) => { return cacacheVerifyStats }, + get: (path, key) => { + if (cacacheEntries[key] === undefined + || cacacheContent[cacacheEntries[key].integrity] === undefined) + throw new Error() + return cacacheContent[cacacheEntries[key].integrity] + }, + rm: { + entry: (path, key) => { + if (cacacheEntries[key] === undefined) + throw new Error() + delete cacacheEntries[key] + }, + content: (path, sha) => { + delete cacacheContent[sha] + }, + }, + ls: (path) => { + return cacacheEntries + }, } const Cache = t.mock('../../lib/cache.js', { @@ -61,6 +150,11 @@ const npm = mockNpm({ output: (msg) => { outputOutput.push(msg) }, + log: { + warn: (...args) => { + logOutput.push(['warn', ...args]) + }, + }, }) const cache = new Cache(npm) @@ -94,13 +188,6 @@ t.test('cache clean (force)', t => { }) }) -t.test('cache clean with arg', t => { - cache.exec(['rm', 'pkg'], err => { - t.match(err.message, 'does not accept arguments', 'should throw error') - t.end() - }) -}) - t.test('cache add no arg', t => { t.teardown(() => { logOutput = [] @@ -136,7 +223,7 @@ t.test('cache add pkg only', t => { t.test('cache add multiple pkgs', t => { t.teardown(() => { - logOutput = [] + outputOutput = [] tarballStreamSpec = '' tarballStreamOpts = {} }) @@ -154,6 +241,182 @@ t.test('cache add multiple pkgs', t => { }) }) +t.test('cache ls', t => { + t.teardown(() => { + outputOutput = [] + logOutput = [] + }) + setupCacacheFixture() + cache.exec(['ls'], err => { + t.error(err) + t.strictSame(outputOutput, [ + 'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy/staydown/-/@fritzy/staydown-3.1.1.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy%2fstaydown', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar/npm-expansion/-/@gar/npm-expansion-2.1.0.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar/npm-expansion/-/@gar/npm-expansion-3.0.0-beta.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar%2fnpm-expansion', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/ape-ecs', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/ape-ecs/-/ape-ecs-2.1.7.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/corrupted', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/corrupted/-/corrupted-3.1.0.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/foo', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/foo/-/foo-1.2.3-beta.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-dist', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-dist/-/missing-dist-23.0.0.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-version', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-version/-/missing-version-16.2.0.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/npm', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/npm/-/npm-1.2.0.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.44.1.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.47.0.tgz', + 'make-fetch-happen:request-cache:https://somerepo.github.org/aabbcc/14', + 'make-fetch-happen:request-cache:https://somerepo.github.org/extemporaneously', + ]) + t.end() + }) +}) + +t.test('cache ls pkgs', t => { + t.teardown(() => { + outputOutput = [] + }) + cache.exec(['ls', 'webpack@>4.44.1', 'npm'], err => { + t.error(err) + t.strictSame(outputOutput, [ + 'make-fetch-happen:request-cache:https://registry.npmjs.org/npm', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/npm/-/npm-1.2.0.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.47.0.tgz', + ]) + t.end() + }) +}) + +t.test('cache ls special', t => { + t.teardown(() => { + outputOutput = [] + }) + cache.exec(['ls', 'foo@1.2.3-beta'], err => { + t.error(err) + t.strictSame(outputOutput, [ + 'make-fetch-happen:request-cache:https://registry.npmjs.org/foo', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/foo/-/foo-1.2.3-beta.tgz', + ]) + t.end() + }) +}) + +t.test('cache ls nonpublic registry', t => { + t.teardown(() => { + outputOutput = [] + }) + cache.exec(['ls', 'extemporaneously'], err => { + t.error(err) + t.strictSame(outputOutput, [ + 'make-fetch-happen:request-cache:https://somerepo.github.org/aabbcc/14', + 'make-fetch-happen:request-cache:https://somerepo.github.org/extemporaneously', + ]) + t.end() + }) +}) + +t.test('cache ls tagged', t => { + t.teardown(() => { + outputOutput = [] + }) + cache.exec(['ls', 'webpack@latest'], err => { + t.match(err.message, 'tagged package', 'should throw warning') + t.end() + }) +}) + +t.test('cache ls scoped and scoped slash', t => { + t.teardown(() => { + outputOutput = [] + }) + cache.exec(['ls', '@fritzy/staydown', '@gar/npm-expansion'], err => { + t.error(err) + t.strictSame(outputOutput, [ + 'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy/staydown/-/@fritzy/staydown-3.1.1.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy%2fstaydown', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar/npm-expansion/-/@gar/npm-expansion-2.1.0.tgz', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar%2fnpm-expansion', + ]) + t.end() + }) +}) + +t.test('cache ls corrupted', t => { + t.teardown(() => { + outputOutput = [] + }) + cache.exec(['ls', 'corrupted'], err => { + t.error(err) + t.strictSame(outputOutput, [ + 'make-fetch-happen:request-cache:https://registry.npmjs.org/corrupted', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/corrupted/-/corrupted-3.1.0.tgz', + ]) + t.end() + }) +}) + +t.test('cache ls missing packument dist', t => { + t.teardown(() => { + outputOutput = [] + }) + cache.exec(['ls', 'missing-dist'], err => { + t.error(err) + t.strictSame(outputOutput, [ + 'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-dist', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-dist/-/missing-dist-23.0.0.tgz', + ]) + t.end() + }) +}) + +t.test('cache ls missing packument version not an object', t => { + t.teardown(() => { + outputOutput = [] + }) + cache.exec(['ls', 'missing-version'], err => { + t.error(err) + t.strictSame(outputOutput, [ + 'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-version', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-version/-/missing-version-16.2.0.tgz', + ]) + t.end() + }) +}) + +t.test('cache rm', t => { + t.teardown(() => { + outputOutput = [] + }) + cache.exec(['rm', + 'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.44.1.tgz'], err => { + t.error(err) + t.strictSame(outputOutput, [ + 'Deleted: make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.44.1.tgz', + ]) + t.end() + }) +}) + +t.test('cache rm unfound', t => { + t.teardown(() => { + outputOutput = [] + logOutput = [] + }) + cache.exec(['rm', 'made-up-key'], err => { + t.error(err) + t.strictSame(logOutput, [ + ['warn', 'Not Found: made-up-key'], + ], 'logs correctly') + t.end() + }) +}) + t.test('cache verify', t => { t.teardown(() => { outputOutput = []