From 244c2069fd093f053d3061c85575ac13e72e2454 Mon Sep 17 00:00:00 2001 From: Ruy Adorno Date: Wed, 9 Dec 2020 17:48:05 -0500 Subject: [PATCH] fix: npm search include/exclude - Fixes `npm search --searchexclude=` option - Tweaks `--searchopt` logic - Refactor and cleanup `lib/search.js` - Add `test/lib/search.js` tests - Fixes: https://github.com/npm/statusboard/issues/171 PR-URL: https://github.com/npm/cli/pull/2325 Credit: @ruyadorno Close: #2325 Reviewed-by: @isaacs --- lib/search.js | 73 ++--- package-lock.json | 2 + package.json | 2 + tap-snapshots/test-lib-search.js-TAP.test.js | 20 ++ test/fixtures/libnpmsearch-stream-result.js | 277 +++++++++++++++++++ test/lib/search.js | 193 +++++++++++++ 6 files changed, 533 insertions(+), 34 deletions(-) create mode 100644 tap-snapshots/test-lib-search.js-TAP.test.js create mode 100644 test/fixtures/libnpmsearch-stream-result.js create mode 100644 test/lib/search.js diff --git a/lib/search.js b/lib/search.js index 5c25bd715ba60..4a4f1f5a7e9c9 100644 --- a/lib/search.js +++ b/lib/search.js @@ -1,27 +1,25 @@ 'use strict' -module.exports = exports = search - +const Minipass = require('minipass') const Pipeline = require('minipass-pipeline') - -const npm = require('./npm.js') -const formatPackageStream = require('./search/format-package-stream.js') - const libSearch = require('libnpmsearch') const log = require('npmlog') + +const formatPackageStream = require('./search/format-package-stream.js') +const packageFilter = require('./search/package-filter.js') +const npm = require('./npm.js') const output = require('./utils/output.js') -const usage = require('./utils/usage') +const usageUtil = require('./utils/usage.js') +const completion = require('./utils/completion/none.js') -search.usage = usage( +const usage = usageUtil( 'search', 'npm search [--long] [search terms ...]' ) -search.completion = function (opts, cb) { - cb(null, []) -} +const cmd = (args, cb) => search(args).then(() => cb()).catch(cb) -function search (args, cb) { +const search = async (args) => { const opts = { ...npm.flatOptions, ...npm.flatOptions.search, @@ -30,22 +28,33 @@ function search (args, cb) { } if (opts.include.length === 0) - return cb(new Error('search must be called with arguments')) + throw new Error('search must be called with arguments') // Used later to figure out whether we had any packages go out let anyOutput = false + class FilterStream extends Minipass { + write (pkg) { + if (packageFilter(pkg, opts.include, opts.exclude)) + super.write(pkg) + } + } + + const filterStream = new FilterStream() + // Grab a configured output stream that will spit out packages in the // desired format. - // - // This is a text minipass stream - var outputStream = formatPackageStream({ + const outputStream = formatPackageStream({ args, // --searchinclude options are not highlighted ...opts, }) log.silly('search', 'searching packages') - const p = new Pipeline(libSearch.stream(opts.include, opts), outputStream) + const p = new Pipeline( + libSearch.stream(opts.include, opts), + filterStream, + outputStream + ) p.on('data', chunk => { if (!anyOutput) @@ -53,24 +62,18 @@ function search (args, cb) { output(chunk.toString('utf8')) }) - p.promise().then(() => { - if (!anyOutput && !opts.json && !opts.parseable) - output('No matches found for ' + (args.map(JSON.stringify).join(' '))) + await p.promise() + if (!anyOutput && !opts.json && !opts.parseable) + output('No matches found for ' + (args.map(JSON.stringify).join(' '))) - log.silly('search', 'search completed') - log.clearProgress() - cb(null, {}) - }, err => cb(err)) + log.silly('search', 'search completed') + log.clearProgress() } function prepareIncludes (args, searchopts) { - if (typeof searchopts !== 'string') - searchopts = '' - return searchopts.split(/\s+/).concat(args).map(function (s) { - return s.toLowerCase() - }).filter(function (s) { - return s - }) + return args + .map(s => s.toLowerCase()) + .filter(s => s) } function prepareExcludes (searchexclude) { @@ -80,7 +83,9 @@ function prepareExcludes (searchexclude) { else exclude = [] - return exclude.map(function (s) { - return s.toLowerCase() - }) + return exclude + .map(s => s.toLowerCase()) + .filter(s => s) } + +module.exports = Object.assign(cmd, { completion, usage }) diff --git a/package-lock.json b/package-lock.json index c95cabe201ca2..0929e80400713 100644 --- a/package-lock.json +++ b/package-lock.json @@ -392,6 +392,8 @@ "libnpmteam": "^2.0.2", "libnpmversion": "^1.0.7", "make-fetch-happen": "^8.0.12", + "minipass": "^3.1.3", + "minipass-pipeline": "^1.2.4", "mkdirp": "^1.0.4", "mkdirp-infer-owner": "^2.0.0", "ms": "^2.1.2", diff --git a/package.json b/package.json index 27ca99e70ffd4..92748bfb078dd 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,8 @@ "libnpmteam": "^2.0.2", "libnpmversion": "^1.0.7", "make-fetch-happen": "^8.0.12", + "minipass": "^3.1.3", + "minipass-pipeline": "^1.2.4", "mkdirp": "^1.0.4", "mkdirp-infer-owner": "^2.0.0", "ms": "^2.1.2", diff --git a/tap-snapshots/test-lib-search.js-TAP.test.js b/tap-snapshots/test-lib-search.js-TAP.test.js new file mode 100644 index 0000000000000..4b4dc75ea3e89 --- /dev/null +++ b/tap-snapshots/test-lib-search.js-TAP.test.js @@ -0,0 +1,20 @@ +/* IMPORTANT + * This snapshot file is auto-generated, but designed for humans. + * It should be checked into source control and tracked carefully. + * Re-generate by setting TAP_SNAPSHOT=1 and running tests. + * Make sure to inspect the output below. Do not ignore changes! + */ +'use strict' +exports[`test/lib/search.js TAP empty search results > should have expected search results 1`] = ` +No matches found for "foo" +` + +exports[`test/lib/search.js TAP search --searchexclude --searchopts > should have filtered expected search results 1`] = ` +NAME | AUTHOR | DATE | VERSION | KEYWORDS +foo | =foo | prehistoric | 1.0.0 | +` + +exports[`test/lib/search.js TAP search > should have expected search results 1`] = ` +NAME | AUTHOR | DATE | VERSION | KEYWORDS +libnpm | =nlf… | 2019-07-16 | 3.0.1 | npm api package manager liblibnpmaccess | =nlf… | 2020-11-03 | 4.0.1 | @evocateur/libnpmaccess | =evocateur | 2019-07-16 | 3.1.2 | @evocateur/libnpmpublish | =evocateur | 2019-07-16 | 1.2.2 | libnpmorg | =nlf… | 2020-11-03 | 2.0.1 | libnpm npm package manager api orgs teamslibnpmsearch | =nlf… | 2020-12-08 | 3.1.0 | npm search api libnpmlibnpmteam | =nlf… | 2020-11-03 | 2.0.2 | libnpmhook | =nlf… | 2020-11-03 | 6.0.1 | npm hooks registry npm apilibnpmpublish | =nlf… | 2020-11-03 | 4.0.0 | libnpmfund | =nlf… | 2020-12-08 | 1.0.2 | npm npmcli libnpm cli git fund gitfund@npmcli/map-workspaces | =nlf… | 2020-09-30 | 1.0.1 | npm npmcli libnpm cli workspaces map-workspaceslibnpmversion | =nlf… | 2020-11-04 | 1.0.7 | @types/libnpmsearch | =types | 2019-09-26 | 2.0.1 | +` diff --git a/test/fixtures/libnpmsearch-stream-result.js b/test/fixtures/libnpmsearch-stream-result.js new file mode 100644 index 0000000000000..4d3aca396fbca --- /dev/null +++ b/test/fixtures/libnpmsearch-stream-result.js @@ -0,0 +1,277 @@ +module.exports = [ + { + name: 'libnpm', + scope: 'unscoped', + version: '3.0.1', + description: 'Collection of programmatic APIs for the npm CLI', + keywords: ['npm', 'api', 'package manager', 'lib'], + date: new Date('2019-07-16T17:50:00.572Z'), + links: { + npm: 'https://www.npmjs.com/package/libnpm', + homepage: 'https://github.com/npm/libnpm#readme', + repository: 'https://github.com/npm/libnpm', + bugs: 'https://github.com/npm/libnpm/issues', + }, + author: { name: 'Kat Marchán', email: 'kzm@zkat.tech' }, + publisher: { username: 'isaacs', email: 'i@izs.me' }, + maintainers: [ + { username: 'nlf', email: 'quitlahok@gmail.com' }, + { username: 'ruyadorno', email: 'ruyadorno@hotmail.com' }, + { username: 'darcyclarke', email: 'darcy@darcyclarke.me' }, + { username: 'isaacs', email: 'i@izs.me' }, + ], + }, + { + name: 'libnpmaccess', + scope: 'unscoped', + version: '4.0.1', + description: 'programmatic library for `npm access` commands', + date: new Date('2020-11-03T19:19:00.526Z'), + links: { + npm: 'https://www.npmjs.com/package/libnpmaccess', + homepage: 'https://npmjs.com/package/libnpmaccess', + repository: 'https://github.com/npm/libnpmaccess', + bugs: 'https://github.com/npm/libnpmaccess/issues', + }, + author: { name: 'Kat Marchán', email: 'kzm@sykosomatic.org' }, + publisher: { username: 'nlf', email: 'quitlahok@gmail.com' }, + maintainers: [ + { username: 'nlf', email: 'quitlahok@gmail.com' }, + { username: 'ruyadorno', email: 'ruyadorno@hotmail.com' }, + { username: 'darcyclarke', email: 'darcy@darcyclarke.me' }, + { username: 'isaacs', email: 'i@izs.me' }, + ], + }, + { + name: '@evocateur/libnpmaccess', + scope: 'evocateur', + version: '3.1.2', + description: 'programmatic library for `npm access` commands', + date: new Date('2019-07-16T19:43:33.959Z'), + links: { + npm: 'https://www.npmjs.com/package/%40evocateur%2Flibnpmaccess', + homepage: 'https://npmjs.com/package/@evocateur/libnpmaccess', + repository: 'https://github.com/evocateur/libnpmaccess', + bugs: 'https://github.com/evocateur/libnpmaccess/issues', + }, + author: { name: 'Kat Marchán', email: 'kzm@zkat.tech' }, + publisher: { username: 'evocateur', email: 'daniel.stockman@gmail.com' }, + maintainers: [{ username: 'evocateur', email: 'daniel.stockman@gmail.com' }], + }, + { + name: '@evocateur/libnpmpublish', + scope: 'evocateur', + version: '1.2.2', + description: 'Programmatic API for the bits behind npm publish and unpublish', + date: new Date('2019-07-16T19:40:40.850Z'), + links: { + npm: 'https://www.npmjs.com/package/%40evocateur%2Flibnpmpublish', + homepage: 'https://npmjs.com/package/@evocateur/libnpmpublish', + repository: 'https://github.com/evocateur/libnpmpublish', + bugs: 'https://github.com/evocateur/libnpmpublish/issues', + }, + author: { name: 'Kat Marchán', email: 'kzm@zkat.tech' }, + publisher: { username: 'evocateur', email: 'daniel.stockman@gmail.com' }, + maintainers: [{ username: 'evocateur', email: 'daniel.stockman@gmail.com' }], + }, + { + name: 'libnpmorg', + scope: 'unscoped', + version: '2.0.1', + description: 'Programmatic api for `npm org` commands', + keywords: ['libnpm', 'npm', 'package manager', 'api', 'orgs', 'teams'], + date: new Date('2020-11-03T19:21:57.757Z'), + links: { + npm: 'https://www.npmjs.com/package/libnpmorg', + homepage: 'https://npmjs.com/package/libnpmorg', + repository: 'https://github.com/npm/libnpmorg', + bugs: 'https://github.com/npm/libnpmorg/issues', + }, + author: { name: 'Kat Marchán', email: 'kzm@sykosomatic.org' }, + publisher: { username: 'nlf', email: 'quitlahok@gmail.com' }, + maintainers: [ + { username: 'nlf', email: 'quitlahok@gmail.com' }, + { username: 'ruyadorno', email: 'ruyadorno@hotmail.com' }, + { username: 'darcyclarke', email: 'darcy@darcyclarke.me' }, + { username: 'isaacs', email: 'i@izs.me' }, + ], + }, + { + name: 'libnpmsearch', + scope: 'unscoped', + version: '3.1.0', + description: 'Programmatic API for searching in npm and compatible registries.', + keywords: ['npm', 'search', 'api', 'libnpm'], + date: new Date('2020-12-08T23:54:18.374Z'), + links: { + npm: 'https://www.npmjs.com/package/libnpmsearch', + homepage: 'https://npmjs.com/package/libnpmsearch', + repository: 'https://github.com/npm/libnpmsearch', + bugs: 'https://github.com/npm/libnpmsearch/issues', + }, + author: { name: 'Kat Marchán', email: 'kzm@sykosomatic.org' }, + publisher: { username: 'isaacs', email: 'i@izs.me' }, + maintainers: [ + { username: 'nlf', email: 'quitlahok@gmail.com' }, + { username: 'ruyadorno', email: 'ruyadorno@hotmail.com' }, + { username: 'darcyclarke', email: 'darcy@darcyclarke.me' }, + { username: 'isaacs', email: 'i@izs.me' }, + ], + }, + { + name: 'libnpmteam', + scope: 'unscoped', + version: '2.0.2', + description: 'npm Team management APIs', + date: new Date('2020-11-03T19:24:42.380Z'), + links: { + npm: 'https://www.npmjs.com/package/libnpmteam', + homepage: 'https://npmjs.com/package/libnpmteam', + repository: 'https://github.com/npm/libnpmteam', + bugs: 'https://github.com/npm/libnpmteam/issues', + }, + author: { name: 'Kat Marchán', email: 'kzm@zkat.tech' }, + publisher: { username: 'nlf', email: 'quitlahok@gmail.com' }, + maintainers: [ + { username: 'nlf', email: 'quitlahok@gmail.com' }, + { username: 'ruyadorno', email: 'ruyadorno@hotmail.com' }, + { username: 'darcyclarke', email: 'darcy@darcyclarke.me' }, + { username: 'isaacs', email: 'i@izs.me' }, + ], + }, + { + name: 'libnpmhook', + scope: 'unscoped', + version: '6.0.1', + description: 'programmatic API for managing npm registry hooks', + keywords: ['npm', 'hooks', 'registry', 'npm api'], + date: new Date('2020-11-03T19:20:45.818Z'), + links: { + npm: 'https://www.npmjs.com/package/libnpmhook', + homepage: 'https://github.com/npm/libnpmhook#readme', + repository: 'https://github.com/npm/libnpmhook', + bugs: 'https://github.com/npm/libnpmhook/issues', + }, + author: { name: 'Kat Marchán', email: 'kzm@sykosomatic.org' }, + publisher: { username: 'nlf', email: 'quitlahok@gmail.com' }, + maintainers: [ + { username: 'nlf', email: 'quitlahok@gmail.com' }, + { username: 'ruyadorno', email: 'ruyadorno@hotmail.com' }, + { username: 'darcyclarke', email: 'darcy@darcyclarke.me' }, + { username: 'isaacs', email: 'i@izs.me' }, + ], + }, + { + name: 'libnpmpublish', + scope: 'unscoped', + version: '4.0.0', + description: 'Programmatic API for the bits behind npm publish and unpublish', + date: new Date('2020-11-03T19:13:43.780Z'), + links: { + npm: 'https://www.npmjs.com/package/libnpmpublish', + homepage: 'https://npmjs.com/package/libnpmpublish', + repository: 'https://github.com/npm/libnpmpublish', + bugs: 'https://github.com/npm/libnpmpublish/issues', + }, + author: { name: 'npm Inc.', email: 'support@npmjs.com' }, + publisher: { username: 'nlf', email: 'quitlahok@gmail.com' }, + maintainers: [ + { username: 'nlf', email: 'quitlahok@gmail.com' }, + { username: 'ruyadorno', email: 'ruyadorno@hotmail.com' }, + { username: 'darcyclarke', email: 'darcy@darcyclarke.me' }, + { username: 'isaacs', email: 'i@izs.me' }, + ], + }, + { + name: 'libnpmfund', + scope: 'unscoped', + version: '1.0.2', + description: 'Programmatic API for npm fund', + keywords: [ + 'npm', 'npmcli', + 'libnpm', 'cli', + 'git', 'fund', + 'gitfund', + ], + date: new Date('2020-12-08T23:22:00.213Z'), + links: { + npm: 'https://www.npmjs.com/package/libnpmfund', + homepage: 'https://github.com/npm/libnpmfund#readme', + repository: 'https://github.com/npm/libnpmfund', + bugs: 'https://github.com/npm/libnpmfund/issues', + }, + author: { name: 'npm Inc.', email: 'support@npmjs.com' }, + publisher: { username: 'isaacs', email: 'i@izs.me' }, + maintainers: [ + { username: 'nlf', email: 'quitlahok@gmail.com' }, + { username: 'ruyadorno', email: 'ruyadorno@hotmail.com' }, + { username: 'darcyclarke', email: 'darcy@darcyclarke.me' }, + { username: 'isaacs', email: 'i@izs.me' }, + ], + }, + { + name: '@npmcli/map-workspaces', + scope: 'npmcli', + version: '1.0.1', + description: 'Retrieves a name:pathname Map for a given workspaces config', + keywords: [ + 'npm', + 'npmcli', + 'libnpm', + 'cli', + 'workspaces', + 'map-workspaces', + ], + date: new Date('2020-09-30T15:16:29.017Z'), + links: { + npm: 'https://www.npmjs.com/package/%40npmcli%2Fmap-workspaces', + homepage: 'https://github.com/npm/map-workspaces#readme', + repository: 'https://github.com/npm/map-workspaces', + bugs: 'https://github.com/npm/map-workspaces/issues', + }, + author: { name: 'npm Inc.', email: 'support@npmjs.com' }, + publisher: { username: 'ruyadorno', email: 'ruyadorno@hotmail.com' }, + maintainers: [ + { username: 'nlf', email: 'quitlahok@gmail.com' }, + { username: 'ruyadorno', email: 'ruyadorno@hotmail.com' }, + { username: 'darcyclarke', email: 'darcy@darcyclarke.me' }, + { username: 'isaacs', email: 'i@izs.me' }, + ], + }, + { + name: 'libnpmversion', + scope: 'unscoped', + version: '1.0.7', + description: "library to do the things that 'npm version' does", + date: new Date('2020-11-04T00:21:41.069Z'), + links: { + npm: 'https://www.npmjs.com/package/libnpmversion', + homepage: 'https://github.com/npm/libnpmversion#readme', + repository: 'https://github.com/npm/libnpmversion', + bugs: 'https://github.com/npm/libnpmversion/issues', + }, + author: { + name: 'Isaac Z. Schlueter', + email: 'i@izs.me', + url: 'https://izs.me', + username: 'isaacs', + }, + publisher: { username: 'isaacs', email: 'i@izs.me' }, + maintainers: [ + { username: 'nlf', email: 'quitlahok@gmail.com' }, + { username: 'ruyadorno', email: 'ruyadorno@hotmail.com' }, + { username: 'darcyclarke', email: 'darcy@darcyclarke.me' }, + { username: 'isaacs', email: 'i@izs.me' }, + ], + }, + { + name: '@types/libnpmsearch', + scope: 'types', + version: '2.0.1', + description: 'TypeScript definitions for libnpmsearch', + date: new Date('2019-09-26T22:24:28.713Z'), + links: { npm: 'https://www.npmjs.com/package/%40types%2Flibnpmsearch' }, + publisher: { username: 'types', email: 'ts-npm-types@microsoft.com' }, + maintainers: [{ username: 'types', email: 'ts-npm-types@microsoft.com' }], + }, +] diff --git a/test/lib/search.js b/test/lib/search.js new file mode 100644 index 0000000000000..1dba1250e08f0 --- /dev/null +++ b/test/lib/search.js @@ -0,0 +1,193 @@ +const Minipass = require('minipass') +const t = require('tap') +const requireInject = require('require-inject') +const libnpmsearchResultFixture = + require('../fixtures/libnpmsearch-stream-result.js') + +let result = '' +const flatOptions = { + search: { + exclude: null, + limit: 20, + opts: '', + }, +} +const npm = { flatOptions: { ...flatOptions } } +const npmlog = { + silly () {}, + clearProgress () {}, +} +const libnpmsearch = { + stream () {}, +} +const mocks = { + npmlog, + libnpmsearch, + '../../lib/npm.js': npm, + '../../lib/utils/output.js': (...msg) => { + result += msg.join('\n') + }, + '../../lib/utils/usage.js': () => 'usage instructions', + // '../../lib/search/format-package-stream.js': a => a, +} + +t.afterEach(cb => { + result = '' + npm.flatOptions = flatOptions + cb() +}) + +const search = requireInject('../../lib/search.js', mocks) + +t.test('no args', t => { + search([], err => { + t.match( + err, + /search must be called with arguments/, + 'should throw usage instructions' + ) + t.end() + }) +}) + +t.test('search ', t => { + const src = new Minipass() + src.objectMode = true + const libnpmsearch = { + stream () { + return src + }, + } + + const search = requireInject('../../lib/search.js', { + ...mocks, + libnpmsearch, + }) + + search(['libnpm'], err => { + if (err) + throw err + + t.matchSnapshot(result, 'should have expected search results') + + t.end() + }) + + for (const i of libnpmsearchResultFixture) + src.write(i) + + src.end() +}) + +t.test('search --searchexclude --searchopts', t => { + npm.flatOptions.search = { + ...flatOptions.search, + exclude: '', + } + + const src = new Minipass() + src.objectMode = true + const libnpmsearch = { + stream () { + return src + }, + } + + const search = requireInject('../../lib/search.js', { + ...mocks, + libnpmsearch, + }) + + search(['foo'], err => { + if (err) + throw err + + t.matchSnapshot(result, 'should have filtered expected search results') + + t.end() + }) + + src.write({ + name: 'foo', + scope: 'unscoped', + version: '1.0.0', + description: '', + keywords: [], + date: null, + author: { name: 'Foo', email: 'foo@npmjs.com' }, + publisher: { name: 'Foo', email: 'foo@npmjs.com' }, + maintainers: [ + { username: 'foo', email: 'foo@npmjs.com' }, + ], + }) + src.write({ + name: 'libnpmversion', + scope: 'unscoped', + version: '1.0.0', + description: '', + keywords: [], + date: null, + author: { name: 'Foo', email: 'foo@npmjs.com' }, + publisher: { name: 'Foo', email: 'foo@npmjs.com' }, + maintainers: [ + { username: 'foo', email: 'foo@npmjs.com' }, + ], + }) + + src.end() +}) + +t.test('empty search results', t => { + const src = new Minipass() + src.objectMode = true + const libnpmsearch = { + stream () { + return src + }, + } + + const search = requireInject('../../lib/search.js', { + ...mocks, + libnpmsearch, + }) + + search(['foo'], err => { + if (err) + throw err + + t.matchSnapshot(result, 'should have expected search results') + + t.end() + }) + + src.end() +}) + +t.test('search api response error', t => { + const src = new Minipass() + src.objectMode = true + const libnpmsearch = { + stream () { + return src + }, + } + + const search = requireInject('../../lib/search.js', { + ...mocks, + libnpmsearch, + }) + + search(['foo'], err => { + t.match( + err, + /ERR/, + 'should throw response error' + ) + + t.end() + }) + + src.emit('error', new Error('ERR')) + + src.end() +})