feat: added --install-strategy=linked (#6078)

Co-authored-by: Vincent Bailly <vibailly@microsoft.com> Implements the RFC https://github.com/npm/rfcs/blob/main/accepted/0042-isolated-mode.md Packages are installed in node_modules/.store flat, and linked into the node_modules tree in depth, rather than hoisted.
npm · Jan 25, 2023 · 8d6d851 · 8d6d851
1 parent 6af152b
commit 8d6d851
Show file tree

Hide file tree

Showing 24 changed files with 2,783 additions and 88 deletions.
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
@@ -618,6 +618,7 @@ graph LR;
   npmcli-arborist-->semver;
   npmcli-arborist-->ssri;
   npmcli-arborist-->tap;
+  npmcli-arborist-->tar-stream;
   npmcli-arborist-->tcompare;
   npmcli-arborist-->treeverse;
   npmcli-arborist-->walk-up-path;

diff --git a/lib/utils/config/definitions.js b/lib/utils/config/definitions.js
@@ -1090,14 +1090,14 @@ define('install-links', {
 
 define('install-strategy', {
   default: 'hoisted',
-  type: ['hoisted', 'nested', 'shallow'],
+  type: ['hoisted', 'nested', 'shallow', 'linked'],
   description: `
     Sets the strategy for installing packages in node_modules.
     hoisted (default): Install non-duplicated in top-level, and duplicated as
       necessary within directory structure.
     nested: (formerly --legacy-bundling) install in place, no hoisting.
     shallow (formerly --global-style) only install direct deps at top-level.
-    linked: (coming soon) install in node_modules/.store, link in place,
+    linked: (experimental) install in node_modules/.store, link in place,
       unhoisted.
   `,
   flatten,

diff --git a/node_modules/.gitignore b/node_modules/.gitignore
@@ -43,13 +43,13 @@
 !/are-we-there-yet
 !/are-we-there-yet/node_modules/
 /are-we-there-yet/node_modules/*
-!/are-we-there-yet/node_modules/buffer
 !/are-we-there-yet/node_modules/readable-stream
 !/balanced-match
 !/base64-js
 !/bin-links
 !/binary-extensions
 !/brace-expansion
+!/buffer
 !/builtins
 !/cacache
 !/chalk

diff --git a/...-there-yet/node_modules/buffer/AUTHORS.md → node_modules/buffer/AUTHORS.md b/...-there-yet/node_modules/buffer/AUTHORS.md → node_modules/buffer/AUTHORS.md
diff --git a/...-we-there-yet/node_modules/buffer/LICENSE → node_modules/buffer/LICENSE b/...-we-there-yet/node_modules/buffer/LICENSE → node_modules/buffer/LICENSE
diff --git a/...-there-yet/node_modules/buffer/index.d.ts → node_modules/buffer/index.d.ts b/...-there-yet/node_modules/buffer/index.d.ts → node_modules/buffer/index.d.ts
diff --git a/...we-there-yet/node_modules/buffer/index.js → node_modules/buffer/index.js b/...we-there-yet/node_modules/buffer/index.js → node_modules/buffer/index.js
diff --git a/...here-yet/node_modules/buffer/package.json → node_modules/buffer/package.json b/...here-yet/node_modules/buffer/package.json → node_modules/buffer/package.json
diff --git a/package-lock.json b/package-lock.json
@@ -2776,29 +2776,6 @@
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
-    "node_modules/are-we-there-yet/node_modules/buffer": {
-      "version": "6.0.3",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "inBundle": true,
-      "license": "MIT",
-      "dependencies": {
-        "base64-js": "^1.3.1",
-        "ieee754": "^1.2.1"
-      }
-    },
     "node_modules/are-we-there-yet/node_modules/readable-stream": {
       "version": "4.2.0",
       "inBundle": true,
@@ -2921,6 +2898,12 @@
       "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
       "dev": true
     },
+    "node_modules/b4a": {
+      "version": "1.6.1",
+      "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.1.tgz",
+      "integrity": "sha512-AsKjNhz72yxteo/0EtQEiwkMUgk/tGmycXlbG4g3Ard2/ULtNLUykGOkeK0egmN27h0xMAhb76jYccW+XTBExA==",
+      "dev": true
+    },
     "node_modules/bail": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz",
@@ -3000,6 +2983,32 @@
         "node": ">=10"
       }
     },
+    "node_modules/bl": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-6.0.0.tgz",
+      "integrity": "sha512-Ik9BVIMdcWzSOCpzDv2XpQ4rJ4oZBuk3ck6MgiOv0EopdgtohN2uSCrrLlkH1Jf0KnpZZMBA3D0bUMbCdj/jgA==",
+      "dev": true,
+      "dependencies": {
+        "buffer": "^6.0.3",
+        "inherits": "^2.0.4",
+        "readable-stream": "^4.2.0"
+      }
+    },
+    "node_modules/bl/node_modules/readable-stream": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.3.0.tgz",
+      "integrity": "sha512-MuEnA0lbSi7JS8XM+WNJlWZkHAAdm7gETHdFK//Q/mChGyj2akEFtdLZh32jSdkWGbRwCW9pn6g3LWDdDeZnBQ==",
+      "dev": true,
+      "dependencies": {
+        "abort-controller": "^3.0.0",
+        "buffer": "^6.0.3",
+        "events": "^3.3.0",
+        "process": "^0.11.10"
+      },
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      }
+    },
     "node_modules/boolbase": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
@@ -3054,6 +3063,30 @@
         "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
       }
     },
+    "node_modules/buffer": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
+      "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "inBundle": true,
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.2.1"
+      }
+    },
     "node_modules/buffer-from": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
@@ -4964,6 +4997,12 @@
       "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
       "dev": true
     },
+    "node_modules/fast-fifo": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.1.0.tgz",
+      "integrity": "sha512-Kl29QoNbNvn4nhDsLYjyIAaIqaJB6rBx5p3sL9VjaefJ+eMFBWVZiaoguaoZfzEKr5RhAti0UgM8703akGPJ6g==",
+      "dev": true
+    },
     "node_modules/fast-json-stable-stringify": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
@@ -9845,6 +9884,12 @@
       ],
       "peer": true
     },
+    "node_modules/queue-tick": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz",
+      "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==",
+      "dev": true
+    },
     "node_modules/quick-lru": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-4.0.1.tgz",
@@ -10930,6 +10975,16 @@
         "escodegen": "^1.8.1"
       }
     },
+    "node_modules/streamx": {
+      "version": "2.13.2",
+      "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.13.2.tgz",
+      "integrity": "sha512-+TWqixPhGDXEG9L/XczSbhfkmwAtGs3BJX5QNU6cvno+pOLKeszByWcnaTu6dg8efsTYqR8ZZuXWHhZfgrxMvA==",
+      "dev": true,
+      "dependencies": {
+        "fast-fifo": "^1.1.0",
+        "queue-tick": "^1.0.1"
+      }
+    },
     "node_modules/string_decoder": {
       "version": "1.3.0",
       "inBundle": true,
@@ -13193,6 +13248,17 @@
         "node": ">=10"
       }
     },
+    "node_modules/tar-stream": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.0.0.tgz",
+      "integrity": "sha512-O6OfUKBbQOqAhh6owTWmA730J/yZCYcpmZ1DBj2YX51ZQrt7d7NgzrR+CnO9wP6nt/viWZW2XeXLavX3/ZEbEg==",
+      "dev": true,
+      "dependencies": {
+        "b4a": "^1.6.1",
+        "bl": "^6.0.0",
+        "streamx": "^2.12.5"
+      }
+    },
     "node_modules/tcompare": {
       "version": "5.0.7",
       "resolved": "https://registry.npmjs.org/tcompare/-/tcompare-5.0.7.tgz",
@@ -14351,6 +14417,7 @@
         "minify-registry-metadata": "^3.0.0",
         "nock": "^13.2.0",
         "tap": "^16.3.2",
+        "tar-stream": "^3.0.0",
         "tcompare": "^5.0.6"
       },
       "engines": {

diff --git a/smoke-tests/tap-snapshots/test/index.js.test.cjs b/smoke-tests/tap-snapshots/test/index.js.test.cjs
@@ -55,9 +55,9 @@ npm ERR! npm ci
 npm ERR! 
 npm ERR! Options:
 npm ERR! [-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
-npm ERR! [-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
-npm ERR! [--legacy-bundling] [--global-style]
-npm ERR! [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
+npm ERR! [-E|--save-exact] [-g|--global]
+npm ERR! [--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
+npm ERR! [--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
 npm ERR! [--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
 npm ERR! [--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
 npm ERR! [-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]

diff --git a/tap-snapshots/test/lib/docs.js.test.cjs b/tap-snapshots/test/lib/docs.js.test.cjs
@@ -1163,13 +1163,13 @@ workspaces.
 #### \`install-strategy\`
 
 * Default: "hoisted"
-* Type: "hoisted", "nested", or "shallow"
+* Type: "hoisted", "nested", "shallow", or "linked"
 
 Sets the strategy for installing packages in node_modules. hoisted
 (default): Install non-duplicated in top-level, and duplicated as necessary
 within directory structure. nested: (formerly --legacy-bundling) install in
 place, no hoisting. shallow (formerly --global-style) only install direct
-deps at top-level. linked: (coming soon) install in node_modules/.store,
+deps at top-level. linked: (experimental) install in node_modules/.store,
 link in place, unhoisted.
 
 #### \`json\`
@@ -2621,9 +2621,9 @@ npm ci
 
 Options:
 [-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
-[-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
-[--legacy-bundling] [--global-style]
-[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
+[-E|--save-exact] [-g|--global]
+[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
+[--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
 [--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
 [--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
 [-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]
@@ -2723,7 +2723,7 @@ Usage:
 npm dedupe
 
 Options:
-[--install-strategy <hoisted|nested|shallow>] [--legacy-bundling]
+[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
 [--global-style] [--strict-peer-deps] [--no-package-lock]
 [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]] [--ignore-scripts]
 [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
@@ -2994,7 +2994,7 @@ Usage:
 npm find-dupes
 
 Options:
-[--install-strategy <hoisted|nested|shallow>] [--legacy-bundling]
+[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
 [--global-style] [--strict-peer-deps] [--no-package-lock]
 [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]] [--ignore-scripts]
 [--no-audit] [--no-bin-links] [--no-fund]
@@ -3175,9 +3175,9 @@ npm install [<package-spec> ...]
 
 Options:
 [-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
-[-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
-[--legacy-bundling] [--global-style]
-[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
+[-E|--save-exact] [-g|--global]
+[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
+[--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
 [--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
 [--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
 [-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]
@@ -3222,9 +3222,9 @@ npm install-ci-test
 
 Options:
 [-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
-[-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
-[--legacy-bundling] [--global-style]
-[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
+[-E|--save-exact] [-g|--global]
+[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
+[--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
 [--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
 [--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
 [-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]
@@ -3269,9 +3269,9 @@ npm install-test [<package-spec> ...]
 
 Options:
 [-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
-[-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
-[--legacy-bundling] [--global-style]
-[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
+[-E|--save-exact] [-g|--global]
+[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
+[--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
 [--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
 [--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
 [-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]
@@ -3316,8 +3316,9 @@ npm link [<package-spec>]
 
 Options:
 [-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
-[-E|--save-exact] [-g|--global] [--install-strategy <hoisted|nested|shallow>]
-[--legacy-bundling] [--global-style] [--strict-peer-deps] [--no-package-lock]
+[-E|--save-exact] [-g|--global]
+[--install-strategy <hoisted|nested|shallow|linked>] [--legacy-bundling]
+[--global-style] [--strict-peer-deps] [--no-package-lock]
 [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]] [--ignore-scripts]
 [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
 [-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]
@@ -4221,8 +4222,9 @@ npm update [<pkg>...]
 
 Options:
 [-S|--save|--no-save|--save-prod|--save-dev|--save-optional|--save-peer|--save-bundle]
-[-g|--global] [--install-strategy <hoisted|nested|shallow>] [--legacy-bundling]
-[--global-style] [--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
+[-g|--global] [--install-strategy <hoisted|nested|shallow|linked>]
+[--legacy-bundling] [--global-style]
+[--omit <dev|optional|peer> [--omit <dev|optional|peer> ...]]
 [--strict-peer-deps] [--no-package-lock] [--foreground-scripts]
 [--ignore-scripts] [--no-audit] [--no-bin-links] [--no-fund] [--dry-run]
 [-w|--workspace <workspace-name> [-w|--workspace <workspace-name> ...]]

diff --git a/workspaces/arborist/lib/arborist/build-ideal-tree.js b/workspaces/arborist/lib/arborist/build-ideal-tree.js
@@ -1232,6 +1232,7 @@ This is a one-time fix-up, please be patient...
     const isWorkspace = this.idealTree.workspaces && this.idealTree.workspaces.has(spec.name)
 
     // spec is a directory, link it unless installLinks is set or it's a workspace
+    // TODO post arborist refactor, will need to check for installStrategy=linked
     if (spec.type === 'directory' && (isWorkspace || !installLinks)) {
       return this[_linkFromSpec](name, spec, parent, edge)
     }

diff --git a/workspaces/arborist/lib/arborist/index.js b/workspaces/arborist/lib/arborist/index.js
@@ -42,6 +42,7 @@ const mixins = [
   require('./load-virtual.js'),
   require('./rebuild.js'),
   require('./reify.js'),
+  require('./isolated-reifier.js'),
 ]
 
 const _workspacesEnabled = Symbol.for('workspacesEnabled')