feat: add npm package-lock.json support (#462)

Co-authored-by: tianding.wk <tianding.wk@alipay.com>

.gitignore

@@ -31,3 +31,14 @@ test/fixtures/high-speed-store/tmp
 .npminstall_tarball
 
 .editorconfig
+.DS_Store
+test/fixtures/flatten/package.json
+test/fixtures/forbidden-license/package.json
+test/fixtures/initial-cnpmrc/package.json
+test/fixtures/local/package.json
+test/fixtures/npm-workspaces/packages/c/
+test/fixtures/npm-workspaces/package.json
+test/fixtures/uninstall/package.json
+
+package-lock.json
+!test/fixtures/lockfile/package-lock.json

bin/install.js

@@ -21,6 +21,7 @@ const {
   ALIAS_TYPES,
 } = require('../lib/npa_types');
 const Context = require('../lib/context');
+const { lockfileConverter } = require('../lib/lockfile_resolver');
 
 const originalArgv = process.argv.slice(2);
 
@@ -39,6 +40,18 @@ Object.assign(argv, parseArgs(originalArgv, {
     // --high-speed-store=filepath
     'high-speed-store',
     'dependencies-tree',
+    /**
+     * set package-lock.json path
+     *
+     * 1. only support package lock v2 and v3.
+     * 2. npminstall doesn't inspect <cwd>/package-lock.json by default.
+     * 3. because arborist doesn't support client/build/isomorphic dependencies,
+     *    these kinds of dependencies will all be ignored.
+     * 4. this option doesn't do extra check for the equivalence of package-lock.json and package.json
+     *    simply behaves like `npm ci` but doesn't remove the node_modules in advance.
+     * 5. you're not supposed to install extra dependencies along with a lockfile.
+     */
+    'lockfile-path',
   ],
   boolean: [
     'version',
@@ -111,6 +124,7 @@ Usage:
   npminstall <git:// url>
   npminstall <github username>/<github project>
   npminstall --proxy=http://localhost:8080
+  npminstall --lockfile-path=</path/to/package-lock.json>
 
 Can specify one or more: npminstall ./foo.tgz bar@stable /some/folder
 If no argument is supplied, installs dependencies from ./package.json.
@@ -304,6 +318,16 @@ debug('argv: %j, env: %j', argv, env);
     };
   }
 
+  const lockfilePath = argv['lockfile-path'];
+  if (lockfilePath) {
+    try {
+      const lockfileData = await fs.readFile(lockfilePath, 'utf8');
+      config.dependenciesTree = lockfileConverter(JSON.parse(lockfileData));
+    } catch (error) {
+      console.warn(chalk.yellow('npminstall WARN load lockfile from %s error :%s'), lockfilePath, error.message);
+    }
+  }
+
   const dependenciesTree = argv['dependencies-tree'];
   if (dependenciesTree) {
     try {

lib/download/npm.js

@@ -730,7 +730,9 @@ const defaultExtensions = toMap([
 function checkShasumAndUngzip(ungzipDir, readstream, pkg, useTarFormat, options) {
   return new Promise((resolve, reject) => {
     const shasum = pkg.dist.shasum;
-    const hash = crypto.createHash('sha1');
+    const integrity = pkg.dist.integrity;
+    const algorithmType = pkg.dist.checkSSRI ? 'sha512' : 'sha1';
+    const hash = crypto.createHash(algorithmType);
     let tarballSize = 0;
     const opts = {
       cwd: ungzipDir,
@@ -800,9 +802,17 @@ function checkShasumAndUngzip(ungzipDir, readstream, pkg, useTarFormat, options)
     });
     readstream.on('end', () => {
       // this will be fire before extracter `env` event fire.
-      const realShasum = hash.digest('hex');
-      if (realShasum !== shasum) {
-        const err = new Error(`real sha1:${realShasum} not equal to remote:${shasum}, download url ${readstream.tarballUrl || ''}, download size ${tarballSize}`);
+      let hashResult = '';
+      let hashString = '';
+      if (pkg.dist.checkSSRI) {
+        hashResult = algorithmType + '-' + hash.digest('base64');
+        hashString = integrity;
+      } else {
+        hashResult = hash.digest('hex');
+        hashString = shasum;
+      }
+      if (hashResult !== hashString) {
+        const err = new Error(`real ${algorithmType}:${hashResult} not equal to remote:${hashString}, download url ${readstream.tarballUrl || ''}, download size ${tarballSize}`);
         err.name = 'ShasumNotMatchError';
         handleCallback(err);
       }

lib/lockfile_resolver.js

@@ -0,0 +1,94 @@
+'use strict';
+
+const path = require('node:path');
+const assert = require('node:assert');
+const dependencies = require('./dependencies');
+
+const NODE_MODULES_DIR = 'node_modules/';
+
+/**
+ * The lockfileConverter converts a npm package-lockfile.json to npminstall .dependencies-tree.json.
+ * Only lockfileVersion >= 2 is supported.
+ * The `.dependencies-tree.json` does not recognize a npm-workspaces package, so we don't need to handle it neither for now.
+ * @param {Object} lockfile package-lock.json data
+ * @param {Object} options installation options
+ * @param {Nested} nested Nested
+ */
+exports.lockfileConverter = function lockfileConverter(lockfile, options, nested) {
+  assert(lockfile.lockfileVersion >= 2, 'Only lockfileVersion >=2 is supported.');
+
+  const tree = {};
+
+  const packages = lockfile.packages;
+  for (const pkgPath in packages) {
+    /**
+     * the lockfile contains all the deps so there's no need to be deps type sensitive.
+     */
+    const deps = dependencies(packages[pkgPath], options, nested);
+    const allMap = deps.allMap;
+    for (const key in allMap) {
+      const mani = exports.nodeModulesPath(pkgPath, key, packages);
+      const dist = {
+        checkSSRI: true,
+        integrity: mani.integrity,
+        tarball: mani.resolved,
+      };
+      // we need to remove the integrity and resolved field from the mani
+      // but the mani should be left intact
+      const maniClone = Object.assign({}, mani);
+      delete maniClone.integrity;
+      delete maniClone.resolved;
+      tree[`${key}@${allMap[key]}`] = {
+        name: key,
+        ...maniClone,
+        dist,
+        _id: `${key}@${mani.version}`,
+      };
+    }
+  }
+
+  return tree;
+};
+
+/**
+ * find the matched version of current semver based on nodejs modules resolution algorithm
+ * we need check the current directory and ancestors directories to find the matched version of lodash.has
+ * e.g.
+ *  "": {
+ *    "dependencies": {
+ *      "lodash.has": "4",
+ *      "a": "latest"
+ *    },
+ *  'node_modules/lodash.has': {
+ *    "version": '4.4.0'
+ *  },
+ *  'node_modules/a': {
+ *    'dependencies': {
+ *       "lodash.has": "3"
+ *    }
+ *  },
+ *  'node_modules/a/node_modules/lodash.has': {
+ *    "version": "3.0.0"
+ *  },
+ * }
+ *
+ * the root lodash.has@4 should matches node_modules/lodash.has
+ *
+ * @param {string} currentPath the current pakcage.json path
+ * @param {string} name the dependency name of current package.json
+ * @param {Object} packages the lockfile packages
+ */
+exports.nodeModulesPath = function nodeModulesPath(currentPath, name, packages) {
+  const dirs = currentPath.split(NODE_MODULES_DIR);
+  dirs.push(name);
+
+  do {
+    const dir = path.normalize('.' + dirs.join('/' + NODE_MODULES_DIR));
+    const pkg = packages[dir];
+    if (pkg) {
+      return pkg;
+    }
+
+    dirs.splice(dirs.length - 2, 1);
+  } while (dirs.length > 1);
+};

test/fixtures/lockfile/package.json

@@ -0,0 +1,9 @@
+{
+  "dependencies": {
+    "lodash.has": "4.0.0",
+    "lodash.get": "3",
+    "lodash._baseget": "3.7.1",
+    "lodash._baseslice": "^3.0.1",
+    "lodash.has3": "npm:lodash.has@3"
+  }
+}

test/install-with-lockfile.test.js

@@ -0,0 +1,48 @@
+const coffee = require('coffee');
+const path = require('node:path');
+const assert = require('node:assert');
+const fs = require('node:fs/promises');
+const { lockfileConverter } = require('../lib/lockfile_resolver');
+const Nested = require('../lib/nested');
+const helper = require('./helper');
+
+describe('test/install-with-lockfile.test.js', () => {
+  const cwd = helper.fixtures('lockfile');
+  const lockfile = require(path.join(cwd, 'package-lock.json'));
+  const nested = new Nested([]);
+  const cleanup = helper.cleanup(cwd);
+
+  beforeEach(cleanup);
+  afterEach(cleanup);
+
+  // the Windows path sucks, shamefully skip these tests
+  if (process.platform !== 'win32') {
+    it('should install successfully', async () => {
+      await coffee.fork(
+        helper.npminstall,
+        [
+          '--lockfile-path',
+          path.join(cwd, 'package-lock.json'),
+        ], { cwd })
+        .debug()
+        .expect('code', 0)
+        .end();
+      assert.strictEqual(
+        await fs.readlink(path.join(cwd, 'node_modules', 'lodash.has3'), 'utf8'),
+        '.store/lodash.has@3.2.1/node_modules/lodash.has'
+      );
+      assert.strictEqual(
+        await fs.readlink(path.join(cwd, 'node_modules', 'lodash.has'), 'utf8'),
+        '.store/lodash.has@4.0.0/node_modules/lodash.has'
+      );
+    });
+
+    it('should convert package-lock.json to .dependencies-tree.json successfully', () => {
+      const dependenciesTree = lockfileConverter(lockfile, {
+        ignoreOptionalDependencies: true,
+      }, nested);
+
+      assert.strictEqual(Object.keys(dependenciesTree).length, 12);
+    });
+  }
+});