From 79f42d3a044b47d3ebf170d7bcc0edfc8fe68c75 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Wed, 8 Jul 2020 15:01:59 +0300 Subject: [PATCH 01/24] feat(plugin-essentials): yarn deduplicate --- .yarn/versions/e9b298c7.yml | 30 ++ .../sources/commands/deduplicate.ts | 283 ++++++++++++++++++ packages/plugin-essentials/sources/index.ts | 2 + packages/yarnpkg-core/sources/StreamReport.ts | 4 + 4 files changed, 319 insertions(+) create mode 100644 .yarn/versions/e9b298c7.yml create mode 100644 packages/plugin-essentials/sources/commands/deduplicate.ts diff --git a/.yarn/versions/e9b298c7.yml b/.yarn/versions/e9b298c7.yml new file mode 100644 index 000000000000..1cd1cdd581e8 --- /dev/null +++ b/.yarn/versions/e9b298c7.yml @@ -0,0 +1,30 @@ +releases: + "@yarnpkg/cli": prerelease + "@yarnpkg/core": prerelease + "@yarnpkg/plugin-essentials": prerelease + +declined: + - "@yarnpkg/plugin-compat" + - "@yarnpkg/plugin-constraints" + - "@yarnpkg/plugin-dlx" + - "@yarnpkg/plugin-exec" + - "@yarnpkg/plugin-file" + - "@yarnpkg/plugin-git" + - "@yarnpkg/plugin-github" + - "@yarnpkg/plugin-http" + - "@yarnpkg/plugin-init" + - "@yarnpkg/plugin-interactive-tools" + - "@yarnpkg/plugin-link" + - "@yarnpkg/plugin-node-modules" + - "@yarnpkg/plugin-npm" + - "@yarnpkg/plugin-npm-cli" + - "@yarnpkg/plugin-pack" + - "@yarnpkg/plugin-patch" + - "@yarnpkg/plugin-pnp" + - "@yarnpkg/plugin-stage" + - "@yarnpkg/plugin-typescript" + - "@yarnpkg/plugin-version" + - "@yarnpkg/plugin-workspace-tools" + - "@yarnpkg/builder" + - "@yarnpkg/doctor" + - "@yarnpkg/pnpify" diff --git a/packages/plugin-essentials/sources/commands/deduplicate.ts b/packages/plugin-essentials/sources/commands/deduplicate.ts new file mode 100644 index 000000000000..a86d45eaca42 --- /dev/null +++ b/packages/plugin-essentials/sources/commands/deduplicate.ts @@ -0,0 +1,283 @@ +/** + * Prior work: + * - https://github.com/atlassian/yarn-deduplicate + * - https://github.com/eps1lon/yarn-plugin-deduplicate + */ + +import {BaseCommand} from '@yarnpkg/cli'; +import {Configuration, Project, ResolveOptions, ThrowReport, Cache, StreamReport, structUtils, IdentHash, LocatorHash, MessageName, Report, DescriptorHash, Locator} from '@yarnpkg/core'; +import {Command} from 'clipanion'; +import micromatch from 'micromatch'; +import semver from 'semver'; + +// eslint-disable-next-line arca/no-default-export +export default class DeduplicateCommand extends BaseCommand { + @Command.Rest() + patterns: Array = []; + + @Command.Boolean(`--check`) + check: boolean = false; + + @Command.Boolean(`--json`) + json: boolean = false; + + static usage = Command.Usage({ + description: `deduplicate dependencies with overlapping ranges`, + details: ` + Duplicates are defined as descriptors with overlapping ranges being resolved and locked to different locators. They are a natural consequence of Yarn's deterministic installs, but they can sometimes pile up and unnecessarily increase the size of your project. + + This command deduplicates dependencies in the current project by reusing (where possible) the locators that most descriptors have been resolved to. This includes both *upgrading* and *downgrading* dependencies where necessary. + + **Note:** Although it never produces a wrong dependency tree, this command should be used with caution, as it modifies the dependency tree, which can sometimes cause problems when packages specify wrong dependency ranges. It is recommended to also review the changes manually. + + If set, the \`--check\` flag will only report the found duplicates, without persisting the modified dependency tree. + + This command accepts glob patterns as arguments (if valid Idents and supported by [micromatch](https://github.com/micromatch/micromatch)). Make sure to escape the patterns, to prevent your own shell from trying to expand them. + `, + examples: [[ + `Deduplicate all packages`, + `$0 deduplicate`, + ], [ + `Deduplicate a specific package`, + `$0 deduplicate lodash`, + ], [ + `Deduplicate all packages with the \`@babel\` scope`, + `$0 deduplicate '@babel/'`, + ], [ + `Check for duplicates (can be used as a CI step)`, + `$0 deduplicate --check`, + ]], + }); + + @Command.Path(`deduplicate`) + async execute() { + const configuration = await Configuration.find(this.context.cwd, this.context.plugins); + const {project} = await Project.find(configuration, this.context.cwd); + const cache = await Cache.find(configuration); + + const deduplicateReport = await StreamReport.start({ + configuration, + includeFooter: false, + stdout: this.context.stdout, + json: this.json, + }, async report => { + await deduplicate(project, this.patterns, {cache, report}); + }); + + if (this.check) { + return deduplicateReport.hasWarnings() ? 1 : 0; + } else { + const installReport = await StreamReport.start({ + configuration, + stdout: this.context.stdout, + json: this.json, + }, async report => { + await project.install({cache, report}); + }); + + return installReport.exitCode(); + } + } +} + +export interface DeduplicationFactors { + locatorsByIdent: Map>; + locatorUsageCounts: Map; +} + +export function getDeduplicationFactors(project: Project): DeduplicationFactors { + const locatorsByIdent = new Map>(); + const locatorUsageCounts = new Map(); + for (const [descriptorHash, locatorHash] of project.storedResolutions) { + const descriptor = project.storedDescriptors.get(descriptorHash); + if (!descriptor) + throw new Error(`Assertion failed: The descriptor (${descriptorHash}) should have been registered`); + + if (!locatorsByIdent.has(descriptor.identHash)) + locatorsByIdent.set(descriptor.identHash, new Set()); + locatorsByIdent.get(descriptor.identHash)!.add(locatorHash); + + const currentUsageCount = locatorUsageCounts.get(locatorHash); + locatorUsageCounts.set(locatorHash, (currentUsageCount ?? 0) + 1); + } + + return {locatorsByIdent, locatorUsageCounts}; +} + +export async function deduplicate(project: Project, patterns: Array, {cache, report}: {cache: Cache, report: Report}) { + const {configuration} = project; + const throwReport = new ThrowReport(); + + const resolver = configuration.makeResolver(); + const fetcher = configuration.makeFetcher(); + + const resolveOptions: ResolveOptions = { + project, + resolver, + report: throwReport, + fetchOptions: { + project, + report: throwReport, + fetcher, + cache, + checksums: project.storedChecksums, + skipIntegrityCheck: true, + }, + }; + + await report.startTimerPromise(`Deduplication step`, async () => { + // We deduplicate in multiple passes because deduplicating a package can cause + // its dependencies - if unused - to be removed from the project.storedDescriptors + // map, which, in turn, can unlock more deduplication possibilities + + let currentPassIdents: Array = []; + let nextPassIdents: Array = [...project.storedDescriptors.values()].map(({identHash}) => identHash); + + // We cache the resolved candidates to speed up subsequent iterations + const candidateCache = new Map>(); + + while (nextPassIdents.length > 0) { + // We resolve the dependency tree between passes to get rid of unused dependencies + await project.resolveEverything({cache, resolver, report: throwReport, lockfileOnly: false}); + + currentPassIdents = nextPassIdents; + nextPassIdents = []; + + const progress = StreamReport.progressViaCounter(project.storedDescriptors.size); + report.reportProgress(progress); + + // We can deduplicate descriptors in parallel (which is 4x faster) because, + // even though we work on the same project instance, we only update their + // resolutions; race conditions are possible when computing the deduplication + // factors (the computed best deduplication candidate not being the best anymore), + // but, because we deduplicate in multiple passes, these problems will solve + // themselves in the next iterations + await Promise.all( + [...project.storedDescriptors.entries()].map(async ([descriptorHash, descriptor]) => { + try { + if (structUtils.isVirtualDescriptor(descriptor)) + return; + + if (!currentPassIdents.includes(descriptor.identHash)) + return; + + if (patterns.length && !micromatch.isMatch(structUtils.stringifyIdent(descriptor), patterns)) + return; + + const currentResolution = project.storedResolutions.get(descriptorHash); + if (!currentResolution) + return; + + // We only care about resolutions that are stored in the lockfile + const currentPackage = project.originalPackages.get(currentResolution); + if (!currentPackage) + return; + + // No need to try deduplicating packages that are not persisted, + // because they will be resolved again anyways + if (!resolver.shouldPersistResolution(currentPackage, resolveOptions)) + return; + + const {locatorsByIdent, locatorUsageCounts} = getDeduplicationFactors(project); + + const locators = locatorsByIdent.get(descriptor.identHash); + if (!locators) + return; + + // No need to choose when there's only one possibility + if (locators.size === 1) + return; + + const sortedLocators = [...locators].sort((a, b) => { + const aUsageCount = locatorUsageCounts.get(a)!; + const bUsageCount = locatorUsageCounts.get(b)!; + + if (aUsageCount !== bUsageCount) + return aUsageCount - bUsageCount; + + const aPackage = project.storedPackages.get(a); + const bPackage = project.storedPackages.get(b); + + if (!aPackage?.version && !bPackage?.version) + return 0; + + if (!aPackage?.version) + return -1; + + if (!bPackage?.version) + return 1; + + return semver.compare(aPackage.version, bPackage.version); + }).reverse(); + + const resolutionDependencies = resolver.getResolutionDependencies(descriptor, resolveOptions); + const dependencies = new Map( + resolutionDependencies.map(dependency => { + const resolution = project.storedResolutions.get(dependency.descriptorHash); + if (!resolution) + throw new Error(`Assertion failed: The resolution (${structUtils.prettyDescriptor(configuration, dependency)}) should have been registered`); + + const pkg = project.storedPackages.get(resolution); + if (!pkg) + throw new Error(`Assertion failed: The package (${resolution}) should have been registered`); + + return [dependency.descriptorHash, pkg] as const; + }) + ); + + let candidates: Array; + if (candidateCache.has(descriptorHash)) { + candidates = candidateCache.get(descriptorHash)!; + } else { + candidates = await resolver.getCandidates(descriptor, dependencies, resolveOptions); + candidateCache.set(descriptorHash, candidates); + } + + const updatedResolution = sortedLocators.find( + locatorHash => candidates.map(({locatorHash}) => locatorHash).includes(locatorHash) + ); + if (!updatedResolution) + return; + + // We only care about resolutions that are stored in the lockfile + const updatedPackage = project.originalPackages.get(updatedResolution); + if (!updatedPackage) + return; + + if (updatedResolution === currentResolution) + return; + + report.reportWarning( + MessageName.UNNAMED, + `${ + structUtils.prettyDescriptor(configuration, descriptor) + } can be deduplicated from ${ + structUtils.prettyLocator(configuration, currentPackage) + } to ${ + structUtils.prettyLocator(configuration, updatedPackage) + }` + ); + + report.reportJson({ + descriptor: structUtils.stringifyDescriptor(descriptor), + currentResolution: structUtils.stringifyLocator(currentPackage), + updatedResolution: structUtils.stringifyLocator(updatedPackage), + }); + + project.storedResolutions.set(descriptorHash, updatedResolution); + + // We schedule some idents for the next pass + nextPassIdents.push( + // We try deduplicating the current package even further + descriptor.identHash, + // We also try deduplicating its dependencies + ...currentPackage.dependencies.keys(), + ); + } finally { + progress.tick(); + } + }) + ); + } + }); +} diff --git a/packages/plugin-essentials/sources/index.ts b/packages/plugin-essentials/sources/index.ts index 01efcc919b5e..6c014e8852ed 100644 --- a/packages/plugin-essentials/sources/index.ts +++ b/packages/plugin-essentials/sources/index.ts @@ -7,6 +7,7 @@ import cleanCache from './commands/cache/clean'; import getConfig from './commands/config/get'; import setConfig from './commands/config/set'; import config from './commands/config'; +import deduplicate from './commands/deduplicate'; import clipanionEntry from './commands/entries/clipanion'; import helpEntry from './commands/entries/help'; import runEntry from './commands/entries/run'; @@ -86,6 +87,7 @@ const plugin: Plugin = { add, bin, config, + deduplicate, exec, install, link, diff --git a/packages/yarnpkg-core/sources/StreamReport.ts b/packages/yarnpkg-core/sources/StreamReport.ts index 2900a6656c07..2cc740336534 100644 --- a/packages/yarnpkg-core/sources/StreamReport.ts +++ b/packages/yarnpkg-core/sources/StreamReport.ts @@ -142,6 +142,10 @@ export class StreamReport extends Report { this.stdout = stdout; } + hasWarnings() { + return this.warningCount > 0; + } + hasErrors() { return this.errorCount > 0; } From f1ce7d2530a5bf7451670935ad5eca6bf707c091 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Wed, 8 Jul 2020 21:02:11 +0300 Subject: [PATCH 02/24] refactor: improve implementation --- .../sources/commands/deduplicate.ts | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/packages/plugin-essentials/sources/commands/deduplicate.ts b/packages/plugin-essentials/sources/commands/deduplicate.ts index a86d45eaca42..c2a9e8c49895 100644 --- a/packages/plugin-essentials/sources/commands/deduplicate.ts +++ b/packages/plugin-essentials/sources/commands/deduplicate.ts @@ -26,7 +26,7 @@ export default class DeduplicateCommand extends BaseCommand { details: ` Duplicates are defined as descriptors with overlapping ranges being resolved and locked to different locators. They are a natural consequence of Yarn's deterministic installs, but they can sometimes pile up and unnecessarily increase the size of your project. - This command deduplicates dependencies in the current project by reusing (where possible) the locators that most descriptors have been resolved to. This includes both *upgrading* and *downgrading* dependencies where necessary. + This command deduplicates dependencies in the current project by reusing (where possible) the locators with the highest versions. This means that dependencies can only be upgraded, never downgraded. **Note:** Although it never produces a wrong dependency tree, this command should be used with caution, as it modifies the dependency tree, which can sometimes cause problems when packages specify wrong dependency ranges. It is recommended to also review the changes manually. @@ -82,12 +82,10 @@ export default class DeduplicateCommand extends BaseCommand { export interface DeduplicationFactors { locatorsByIdent: Map>; - locatorUsageCounts: Map; } export function getDeduplicationFactors(project: Project): DeduplicationFactors { const locatorsByIdent = new Map>(); - const locatorUsageCounts = new Map(); for (const [descriptorHash, locatorHash] of project.storedResolutions) { const descriptor = project.storedDescriptors.get(descriptorHash); if (!descriptor) @@ -96,12 +94,9 @@ export function getDeduplicationFactors(project: Project): DeduplicationFactors if (!locatorsByIdent.has(descriptor.identHash)) locatorsByIdent.set(descriptor.identHash, new Set()); locatorsByIdent.get(descriptor.identHash)!.add(locatorHash); - - const currentUsageCount = locatorUsageCounts.get(locatorHash); - locatorUsageCounts.set(locatorHash, (currentUsageCount ?? 0) + 1); } - return {locatorsByIdent, locatorUsageCounts}; + return {locatorsByIdent}; } export async function deduplicate(project: Project, patterns: Array, {cache, report}: {cache: Cache, report: Report}) { @@ -178,7 +173,7 @@ export async function deduplicate(project: Project, patterns: Array, {ca if (!resolver.shouldPersistResolution(currentPackage, resolveOptions)) return; - const {locatorsByIdent, locatorUsageCounts} = getDeduplicationFactors(project); + const {locatorsByIdent} = getDeduplicationFactors(project); const locators = locatorsByIdent.get(descriptor.identHash); if (!locators) @@ -189,12 +184,6 @@ export async function deduplicate(project: Project, patterns: Array, {ca return; const sortedLocators = [...locators].sort((a, b) => { - const aUsageCount = locatorUsageCounts.get(a)!; - const bUsageCount = locatorUsageCounts.get(b)!; - - if (aUsageCount !== bUsageCount) - return aUsageCount - bUsageCount; - const aPackage = project.storedPackages.get(a); const bPackage = project.storedPackages.get(b); From 4303e0da422594287cef801d3ffb22cffad097fd Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Mon, 17 Aug 2020 03:17:52 +0300 Subject: [PATCH 03/24] refactor: shorten command name to `yarn dedupe` --- .../sources/commands/{deduplicate.ts => dedupe.ts} | 12 ++++++------ packages/plugin-essentials/sources/index.ts | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) rename packages/plugin-essentials/sources/commands/{deduplicate.ts => dedupe.ts} (98%) diff --git a/packages/plugin-essentials/sources/commands/deduplicate.ts b/packages/plugin-essentials/sources/commands/dedupe.ts similarity index 98% rename from packages/plugin-essentials/sources/commands/deduplicate.ts rename to packages/plugin-essentials/sources/commands/dedupe.ts index c2a9e8c49895..970cd15587d7 100644 --- a/packages/plugin-essentials/sources/commands/deduplicate.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -11,7 +11,7 @@ import micromatch import semver from 'semver'; // eslint-disable-next-line arca/no-default-export -export default class DeduplicateCommand extends BaseCommand { +export default class DedupeCommand extends BaseCommand { @Command.Rest() patterns: Array = []; @@ -36,20 +36,20 @@ export default class DeduplicateCommand extends BaseCommand { `, examples: [[ `Deduplicate all packages`, - `$0 deduplicate`, + `$0 dedupe`, ], [ `Deduplicate a specific package`, - `$0 deduplicate lodash`, + `$0 dedupe lodash`, ], [ `Deduplicate all packages with the \`@babel\` scope`, - `$0 deduplicate '@babel/'`, + `$0 dedupe '@babel/*'`, ], [ `Check for duplicates (can be used as a CI step)`, - `$0 deduplicate --check`, + `$0 dedupe --check`, ]], }); - @Command.Path(`deduplicate`) + @Command.Path(`dedupe`) async execute() { const configuration = await Configuration.find(this.context.cwd, this.context.plugins); const {project} = await Project.find(configuration, this.context.cwd); diff --git a/packages/plugin-essentials/sources/index.ts b/packages/plugin-essentials/sources/index.ts index 6c014e8852ed..1abcb5855b8d 100644 --- a/packages/plugin-essentials/sources/index.ts +++ b/packages/plugin-essentials/sources/index.ts @@ -7,7 +7,7 @@ import cleanCache from './commands/cache/clean'; import getConfig from './commands/config/get'; import setConfig from './commands/config/set'; import config from './commands/config'; -import deduplicate from './commands/deduplicate'; +import dedupe from './commands/dedupe'; import clipanionEntry from './commands/entries/clipanion'; import helpEntry from './commands/entries/help'; import runEntry from './commands/entries/run'; @@ -87,7 +87,7 @@ const plugin: Plugin = { add, bin, config, - deduplicate, + dedupe, exec, install, link, From 408acbfbb618e48f43b95066ba45fdf286e847d8 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Mon, 17 Aug 2020 15:55:58 +0300 Subject: [PATCH 04/24] refactor: rewrite algorithm --- .../sources/commands/dedupe.ts | 289 ++++++++---------- 1 file changed, 130 insertions(+), 159 deletions(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index 970cd15587d7..eb403dce6f01 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -4,17 +4,116 @@ * - https://github.com/eps1lon/yarn-plugin-deduplicate */ -import {BaseCommand} from '@yarnpkg/cli'; -import {Configuration, Project, ResolveOptions, ThrowReport, Cache, StreamReport, structUtils, IdentHash, LocatorHash, MessageName, Report, DescriptorHash, Locator} from '@yarnpkg/core'; -import {Command} from 'clipanion'; -import micromatch from 'micromatch'; -import semver from 'semver'; +import {BaseCommand} from '@yarnpkg/cli'; +import {Configuration, Project, ResolveOptions, ThrowReport, Cache, StreamReport, Resolver, miscUtils, Descriptor, Package} from '@yarnpkg/core'; +import {structUtils, IdentHash, LocatorHash, MessageName, Report, Fetcher, FetchOptions} from '@yarnpkg/core'; +import {Command} from 'clipanion'; +import micromatch from 'micromatch'; + +export const dedupeSkip = Symbol(`dedupeSkip`); + +export type DedupePromise = Promise<{ + descriptor: Descriptor, + currentPackage: Package, + updatedPackage: Package, +} | typeof dedupeSkip>; + +export type DedupeAlgorithm = (project: Project, patterns: Array, opts: { + resolver: Resolver, + resolveOptions: ResolveOptions, + fetcher: Fetcher, + fetchOptions: FetchOptions, + report: Report, +}) => Promise>; + +export enum Strategy { + Highest = `highest`, +} + +export const DEDUPE_ALGORITHMS: Record = { + highest: async (project, patterns, {resolver, fetcher, resolveOptions, fetchOptions, report}) => { + const locatorsByIdent = new Map>(); + for (const [descriptorHash, locatorHash] of project.storedResolutions) { + const descriptor = project.storedDescriptors.get(descriptorHash); + if (!descriptor) + throw new Error(`Assertion failed: The descriptor (${descriptorHash}) should have been registered`); + + miscUtils.getSetWithDefault(locatorsByIdent, descriptor.identHash).add(locatorHash); + } + + return Array.from(project.storedDescriptors.values(), async descriptor => { + if (structUtils.isVirtualDescriptor(descriptor)) + return dedupeSkip; + + if (patterns.length && !micromatch.isMatch(structUtils.stringifyIdent(descriptor), patterns)) + return dedupeSkip; + + const currentResolution = project.storedResolutions.get(descriptor.descriptorHash); + if (typeof currentResolution === `undefined`) + return dedupeSkip; + + // We only care about resolutions that are stored in the lockfile + const currentPackage = project.originalPackages.get(currentResolution); + if (typeof currentPackage === `undefined`) + return dedupeSkip; + + // No need to try deduping packages that are not persisted, + // they will be resolved again anyways + if (!resolver.shouldPersistResolution(currentPackage, resolveOptions)) + return dedupeSkip; + + const locators = locatorsByIdent.get(descriptor.identHash); + if (typeof locators === `undefined`) + return dedupeSkip; + + // No need to choose when there's only one possibility + if (locators.size === 1) + return dedupeSkip; + + const resolutionDependencies = resolver.getResolutionDependencies(descriptor, resolveOptions); + const dependencies = new Map( + resolutionDependencies.map(dependency => { + const resolution = project.storedResolutions.get(dependency.descriptorHash); + if (typeof resolution === `undefined`) + throw new Error(`Assertion failed: The resolution (${structUtils.prettyDescriptor(project.configuration, dependency)}) should have been registered`); + + const pkg = project.storedPackages.get(resolution); + if (typeof pkg === `undefined`) + throw new Error(`Assertion failed: The package (${resolution}) should have been registered`); + + return [dependency.descriptorHash, pkg] as const; + }) + ); + + const candidates = await resolver.getCandidates(descriptor, dependencies, resolveOptions); + + const bestCandidate = candidates.find(({locatorHash}) => locators.has(locatorHash)); + if (typeof bestCandidate === `undefined`) + return dedupeSkip; + + const updatedResolution = bestCandidate.locatorHash; + + // We only care about resolutions that are stored in the lockfile + const updatedPackage = project.originalPackages.get(updatedResolution); + if (typeof updatedPackage === `undefined`) + return dedupeSkip; + + if (updatedResolution === currentResolution) + return dedupeSkip; + + return {descriptor, currentPackage, updatedPackage}; + }); + }, +}; // eslint-disable-next-line arca/no-default-export export default class DedupeCommand extends BaseCommand { @Command.Rest() patterns: Array = []; + @Command.String(`-s,--strategy`) + strategy: Strategy = Strategy.Highest; + @Command.Boolean(`--check`) check: boolean = false; @@ -61,7 +160,7 @@ export default class DedupeCommand extends BaseCommand { stdout: this.context.stdout, json: this.json, }, async report => { - await deduplicate(project, this.patterns, {cache, report}); + await deduplicate(this.strategy, project, this.patterns, {cache, report}); }); if (this.check) { @@ -80,161 +179,43 @@ export default class DedupeCommand extends BaseCommand { } } -export interface DeduplicationFactors { - locatorsByIdent: Map>; -} - -export function getDeduplicationFactors(project: Project): DeduplicationFactors { - const locatorsByIdent = new Map>(); - for (const [descriptorHash, locatorHash] of project.storedResolutions) { - const descriptor = project.storedDescriptors.get(descriptorHash); - if (!descriptor) - throw new Error(`Assertion failed: The descriptor (${descriptorHash}) should have been registered`); - - if (!locatorsByIdent.has(descriptor.identHash)) - locatorsByIdent.set(descriptor.identHash, new Set()); - locatorsByIdent.get(descriptor.identHash)!.add(locatorHash); - } - - return {locatorsByIdent}; -} - -export async function deduplicate(project: Project, patterns: Array, {cache, report}: {cache: Cache, report: Report}) { +export async function deduplicate(strategy: Strategy, project: Project, patterns: Array, {cache, report}: {cache: Cache, report: Report}) { const {configuration} = project; const throwReport = new ThrowReport(); const resolver = configuration.makeResolver(); const fetcher = configuration.makeFetcher(); + const fetchOptions: FetchOptions = { + cache, + checksums: project.storedChecksums, + fetcher, + project, + report: throwReport, + skipIntegrityCheck: true, + }; const resolveOptions: ResolveOptions = { project, resolver, report: throwReport, - fetchOptions: { - project, - report: throwReport, - fetcher, - cache, - checksums: project.storedChecksums, - skipIntegrityCheck: true, - }, + fetchOptions, }; await report.startTimerPromise(`Deduplication step`, async () => { - // We deduplicate in multiple passes because deduplicating a package can cause - // its dependencies - if unused - to be removed from the project.storedDescriptors - // map, which, in turn, can unlock more deduplication possibilities - - let currentPassIdents: Array = []; - let nextPassIdents: Array = [...project.storedDescriptors.values()].map(({identHash}) => identHash); - - // We cache the resolved candidates to speed up subsequent iterations - const candidateCache = new Map>(); - - while (nextPassIdents.length > 0) { - // We resolve the dependency tree between passes to get rid of unused dependencies - await project.resolveEverything({cache, resolver, report: throwReport, lockfileOnly: false}); - - currentPassIdents = nextPassIdents; - nextPassIdents = []; - - const progress = StreamReport.progressViaCounter(project.storedDescriptors.size); - report.reportProgress(progress); - - // We can deduplicate descriptors in parallel (which is 4x faster) because, - // even though we work on the same project instance, we only update their - // resolutions; race conditions are possible when computing the deduplication - // factors (the computed best deduplication candidate not being the best anymore), - // but, because we deduplicate in multiple passes, these problems will solve - // themselves in the next iterations - await Promise.all( - [...project.storedDescriptors.entries()].map(async ([descriptorHash, descriptor]) => { - try { - if (structUtils.isVirtualDescriptor(descriptor)) - return; - - if (!currentPassIdents.includes(descriptor.identHash)) - return; - - if (patterns.length && !micromatch.isMatch(structUtils.stringifyIdent(descriptor), patterns)) - return; - - const currentResolution = project.storedResolutions.get(descriptorHash); - if (!currentResolution) - return; - - // We only care about resolutions that are stored in the lockfile - const currentPackage = project.originalPackages.get(currentResolution); - if (!currentPackage) - return; - - // No need to try deduplicating packages that are not persisted, - // because they will be resolved again anyways - if (!resolver.shouldPersistResolution(currentPackage, resolveOptions)) - return; - - const {locatorsByIdent} = getDeduplicationFactors(project); - - const locators = locatorsByIdent.get(descriptor.identHash); - if (!locators) - return; - - // No need to choose when there's only one possibility - if (locators.size === 1) - return; - - const sortedLocators = [...locators].sort((a, b) => { - const aPackage = project.storedPackages.get(a); - const bPackage = project.storedPackages.get(b); + const algorithm = DEDUPE_ALGORITHMS[strategy]; + const dedupePromises = await algorithm(project, patterns, {resolver, resolveOptions, fetcher, fetchOptions, report}); - if (!aPackage?.version && !bPackage?.version) - return 0; + const progress = StreamReport.progressViaCounter(dedupePromises.length); + report.reportProgress(progress); - if (!aPackage?.version) - return -1; - - if (!bPackage?.version) - return 1; - - return semver.compare(aPackage.version, bPackage.version); - }).reverse(); - - const resolutionDependencies = resolver.getResolutionDependencies(descriptor, resolveOptions); - const dependencies = new Map( - resolutionDependencies.map(dependency => { - const resolution = project.storedResolutions.get(dependency.descriptorHash); - if (!resolution) - throw new Error(`Assertion failed: The resolution (${structUtils.prettyDescriptor(configuration, dependency)}) should have been registered`); - - const pkg = project.storedPackages.get(resolution); - if (!pkg) - throw new Error(`Assertion failed: The package (${resolution}) should have been registered`); - - return [dependency.descriptorHash, pkg] as const; - }) - ); - - let candidates: Array; - if (candidateCache.has(descriptorHash)) { - candidates = candidateCache.get(descriptorHash)!; - } else { - candidates = await resolver.getCandidates(descriptor, dependencies, resolveOptions); - candidateCache.set(descriptorHash, candidates); - } - - const updatedResolution = sortedLocators.find( - locatorHash => candidates.map(({locatorHash}) => locatorHash).includes(locatorHash) - ); - if (!updatedResolution) + await Promise.all( + dedupePromises.map(dedupePromise => + dedupePromise + .then(dedupe => { + if (dedupe === dedupeSkip) return; - // We only care about resolutions that are stored in the lockfile - const updatedPackage = project.originalPackages.get(updatedResolution); - if (!updatedPackage) - return; - - if (updatedResolution === currentResolution) - return; + const {descriptor, currentPackage, updatedPackage} = dedupe; report.reportWarning( MessageName.UNNAMED, @@ -253,20 +234,10 @@ export async function deduplicate(project: Project, patterns: Array, {ca updatedResolution: structUtils.stringifyLocator(updatedPackage), }); - project.storedResolutions.set(descriptorHash, updatedResolution); - - // We schedule some idents for the next pass - nextPassIdents.push( - // We try deduplicating the current package even further - descriptor.identHash, - // We also try deduplicating its dependencies - ...currentPackage.dependencies.keys(), - ); - } finally { - progress.tick(); - } - }) - ); - } + project.storedResolutions.set(descriptor.descriptorHash, updatedPackage.locatorHash); + }) + .finally(() => progress.tick()) + ) + ); }); } From 7063670e051302653562b47268af5ec3166c08ff Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Mon, 17 Aug 2020 16:58:50 +0300 Subject: [PATCH 05/24] refactor: more refactoring --- .pnp.js | 3 + .yarn/versions/e9b298c7.yml | 12 +-- packages/plugin-essentials/package.json | 1 + .../sources/commands/dedupe.ts | 83 +++++++++++++++---- packages/yarnpkg-core/sources/StreamReport.ts | 4 - yarn.lock | 1 + 6 files changed, 72 insertions(+), 32 deletions(-) diff --git a/.pnp.js b/.pnp.js index 547d46c3cee9..32aaa56078c8 100755 --- a/.pnp.js +++ b/.pnp.js @@ -9153,6 +9153,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@types/treeify", "npm:1.0.0"], ["@types/yarnpkg__cli", null], ["@types/yarnpkg__core", null], + ["@types/yup", "npm:0.26.12"], ["@yarnpkg/cli", "workspace:packages/yarnpkg-cli"], ["@yarnpkg/core", "workspace:packages/yarnpkg-core"], ["@yarnpkg/fslib", "workspace:packages/yarnpkg-fslib"], @@ -9187,6 +9188,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@types/treeify", "npm:1.0.0"], ["@types/yarnpkg__cli", null], ["@types/yarnpkg__core", null], + ["@types/yup", "npm:0.26.12"], ["@yarnpkg/cli", "virtual:e04a2594c769771b96db34e7a92a8a3af1c98ae86dce662589a5c5d5209e16875506f8cb5f4c2230a2b2ae06335b14466352c4ed470d39edf9edb6c515984525#workspace:packages/yarnpkg-cli"], ["@yarnpkg/core", "workspace:packages/yarnpkg-core"], ["@yarnpkg/fslib", "workspace:packages/yarnpkg-fslib"], @@ -9219,6 +9221,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@types/micromatch", "npm:4.0.1"], ["@types/semver", "npm:7.1.0"], ["@types/treeify", "npm:1.0.0"], + ["@types/yup", "npm:0.26.12"], ["@yarnpkg/cli", "virtual:e04a2594c769771b96db34e7a92a8a3af1c98ae86dce662589a5c5d5209e16875506f8cb5f4c2230a2b2ae06335b14466352c4ed470d39edf9edb6c515984525#workspace:packages/yarnpkg-cli"], ["@yarnpkg/core", "workspace:packages/yarnpkg-core"], ["@yarnpkg/fslib", "workspace:packages/yarnpkg-fslib"], diff --git a/.yarn/versions/e9b298c7.yml b/.yarn/versions/e9b298c7.yml index 1cd1cdd581e8..e9b059fffe3b 100644 --- a/.yarn/versions/e9b298c7.yml +++ b/.yarn/versions/e9b298c7.yml @@ -1,22 +1,14 @@ releases: "@yarnpkg/cli": prerelease - "@yarnpkg/core": prerelease - "@yarnpkg/plugin-essentials": prerelease + "@yarnpkg/plugin-essentials": minor declined: - "@yarnpkg/plugin-compat" - "@yarnpkg/plugin-constraints" - "@yarnpkg/plugin-dlx" - - "@yarnpkg/plugin-exec" - - "@yarnpkg/plugin-file" - - "@yarnpkg/plugin-git" - - "@yarnpkg/plugin-github" - - "@yarnpkg/plugin-http" - "@yarnpkg/plugin-init" - "@yarnpkg/plugin-interactive-tools" - - "@yarnpkg/plugin-link" - "@yarnpkg/plugin-node-modules" - - "@yarnpkg/plugin-npm" - "@yarnpkg/plugin-npm-cli" - "@yarnpkg/plugin-pack" - "@yarnpkg/plugin-patch" @@ -26,5 +18,5 @@ declined: - "@yarnpkg/plugin-version" - "@yarnpkg/plugin-workspace-tools" - "@yarnpkg/builder" + - "@yarnpkg/core" - "@yarnpkg/doctor" - - "@yarnpkg/pnpify" diff --git a/packages/plugin-essentials/package.json b/packages/plugin-essentials/package.json index b4b56dbf7e2b..f57e66617e71 100644 --- a/packages/plugin-essentials/package.json +++ b/packages/plugin-essentials/package.json @@ -27,6 +27,7 @@ "@types/micromatch": "^4.0.1", "@types/semver": "^7.1.0", "@types/treeify": "^1.0.0", + "@types/yup": "0.26.12", "@yarnpkg/cli": "workspace:^2.1.1", "@yarnpkg/core": "workspace:^2.1.1" }, diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index eb403dce6f01..b6e4c5b44603 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -4,11 +4,12 @@ * - https://github.com/eps1lon/yarn-plugin-deduplicate */ -import {BaseCommand} from '@yarnpkg/cli'; -import {Configuration, Project, ResolveOptions, ThrowReport, Cache, StreamReport, Resolver, miscUtils, Descriptor, Package} from '@yarnpkg/core'; -import {structUtils, IdentHash, LocatorHash, MessageName, Report, Fetcher, FetchOptions} from '@yarnpkg/core'; -import {Command} from 'clipanion'; -import micromatch from 'micromatch'; +import {BaseCommand} from '@yarnpkg/cli'; +import {Configuration, Project, ResolveOptions, ThrowReport, Cache, StreamReport, Resolver, miscUtils, Descriptor, Package, FormatType} from '@yarnpkg/core'; +import {structUtils, IdentHash, LocatorHash, MessageName, Report, Fetcher, FetchOptions} from '@yarnpkg/core'; +import {Command} from 'clipanion'; +import micromatch from 'micromatch'; +import * as yup from 'yup'; export const dedupeSkip = Symbol(`dedupeSkip`); @@ -23,19 +24,20 @@ export type DedupeAlgorithm = (project: Project, patterns: Array, opts: resolveOptions: ResolveOptions, fetcher: Fetcher, fetchOptions: FetchOptions, - report: Report, }) => Promise>; export enum Strategy { - Highest = `highest`, + HIGHEST = `highest`, } +const acceptedStrategies = new Set(Object.values(Strategy)); + export const DEDUPE_ALGORITHMS: Record = { - highest: async (project, patterns, {resolver, fetcher, resolveOptions, fetchOptions, report}) => { + highest: async (project, patterns, {resolver, fetcher, resolveOptions, fetchOptions}) => { const locatorsByIdent = new Map>(); for (const [descriptorHash, locatorHash] of project.storedResolutions) { const descriptor = project.storedDescriptors.get(descriptorHash); - if (!descriptor) + if (typeof descriptor === `undefined`) throw new Error(`Assertion failed: The descriptor (${descriptorHash}) should have been registered`); miscUtils.getSetWithDefault(locatorsByIdent, descriptor.identHash).add(locatorHash); @@ -112,7 +114,7 @@ export default class DedupeCommand extends BaseCommand { patterns: Array = []; @Command.String(`-s,--strategy`) - strategy: Strategy = Strategy.Highest; + strategy: Strategy = Strategy.HIGHEST; @Command.Boolean(`--check`) check: boolean = false; @@ -120,6 +122,17 @@ export default class DedupeCommand extends BaseCommand { @Command.Boolean(`--json`) json: boolean = false; + static schema = yup.object().shape({ + strategy: yup.string().test({ + name: `strategy`, + message: `\${path} must be one of \${strategies}`, + params: {strategies: [...acceptedStrategies].join(`, `)}, + test: (strategy: string) => { + return acceptedStrategies.has(strategy as Strategy); + }, + }), + }); + static usage = Command.Usage({ description: `deduplicate dependencies with overlapping ranges`, details: ` @@ -154,17 +167,18 @@ export default class DedupeCommand extends BaseCommand { const {project} = await Project.find(configuration, this.context.cwd); const cache = await Cache.find(configuration); - const deduplicateReport = await StreamReport.start({ + let dedupedPackageCount: number = 0; + await StreamReport.start({ configuration, includeFooter: false, stdout: this.context.stdout, json: this.json, }, async report => { - await deduplicate(this.strategy, project, this.patterns, {cache, report}); + dedupedPackageCount = await dedupe({project, strategy: this.strategy, patterns: this.patterns, cache, report}); }); if (this.check) { - return deduplicateReport.hasWarnings() ? 1 : 0; + return dedupedPackageCount ? 1 : 0; } else { const installReport = await StreamReport.start({ configuration, @@ -179,7 +193,15 @@ export default class DedupeCommand extends BaseCommand { } } -export async function deduplicate(strategy: Strategy, project: Project, patterns: Array, {cache, report}: {cache: Cache, report: Report}) { +export type DedupeSpec = { + strategy: Strategy, + project: Project, + patterns: Array, + cache: Cache, + report: Report, +}; + +export async function dedupe({strategy, project, patterns, cache, report}: DedupeSpec) { const {configuration} = project; const throwReport = new ThrowReport(); @@ -201,13 +223,15 @@ export async function deduplicate(strategy: Strategy, project: Project, patterns fetchOptions, }; - await report.startTimerPromise(`Deduplication step`, async () => { + return await report.startTimerPromise(`Deduplication step`, async () => { const algorithm = DEDUPE_ALGORITHMS[strategy]; - const dedupePromises = await algorithm(project, patterns, {resolver, resolveOptions, fetcher, fetchOptions, report}); + const dedupePromises = await algorithm(project, patterns, {resolver, resolveOptions, fetcher, fetchOptions}); const progress = StreamReport.progressViaCounter(dedupePromises.length); report.reportProgress(progress); + let dedupedPackageCount = 0; + await Promise.all( dedupePromises.map(dedupePromise => dedupePromise @@ -215,13 +239,15 @@ export async function deduplicate(strategy: Strategy, project: Project, patterns if (dedupe === dedupeSkip) return; + dedupedPackageCount++; + const {descriptor, currentPackage, updatedPackage} = dedupe; - report.reportWarning( + report.reportInfo( MessageName.UNNAMED, `${ structUtils.prettyDescriptor(configuration, descriptor) - } can be deduplicated from ${ + } can be deduped from ${ structUtils.prettyLocator(configuration, currentPackage) } to ${ structUtils.prettyLocator(configuration, updatedPackage) @@ -239,5 +265,26 @@ export async function deduplicate(strategy: Strategy, project: Project, patterns .finally(() => progress.tick()) ) ); + + const prettyStrategy = configuration.format(strategy, FormatType.CODE); + + let packages: string; + switch (dedupedPackageCount) { + case 0: { + packages = `No packages`; + } break; + + case 1: { + packages = `One package`; + } break; + + default: { + packages = `${dedupedPackageCount} packages`; + } + } + + report.reportInfo(MessageName.UNNAMED, `${packages} can be deduped using the strategy ${prettyStrategy}`); + + return dedupedPackageCount; }); } diff --git a/packages/yarnpkg-core/sources/StreamReport.ts b/packages/yarnpkg-core/sources/StreamReport.ts index 4e95f6b92d5f..d1014631bb2f 100644 --- a/packages/yarnpkg-core/sources/StreamReport.ts +++ b/packages/yarnpkg-core/sources/StreamReport.ts @@ -192,10 +192,6 @@ export class StreamReport extends Report { this.stdout = stdout; } - hasWarnings() { - return this.warningCount > 0; - } - hasErrors() { return this.errorCount > 0; } diff --git a/yarn.lock b/yarn.lock index 07559b67b0b6..585ef02e74df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5715,6 +5715,7 @@ __metadata: "@types/micromatch": ^4.0.1 "@types/semver": ^7.1.0 "@types/treeify": ^1.0.0 + "@types/yup": 0.26.12 "@yarnpkg/cli": "workspace:^2.1.1" "@yarnpkg/core": "workspace:^2.1.1" "@yarnpkg/fslib": "workspace:^2.1.0" From d5a2291e26025d27fbf8700890f9321db644fba1 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Mon, 17 Aug 2020 17:55:44 +0300 Subject: [PATCH 06/24] refactor: even more refactoring --- packages/plugin-essentials/sources/commands/dedupe.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index b6e4c5b44603..da40e780d545 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -168,7 +168,7 @@ export default class DedupeCommand extends BaseCommand { const cache = await Cache.find(configuration); let dedupedPackageCount: number = 0; - await StreamReport.start({ + const dedupeReport = await StreamReport.start({ configuration, includeFooter: false, stdout: this.context.stdout, @@ -177,6 +177,9 @@ export default class DedupeCommand extends BaseCommand { dedupedPackageCount = await dedupe({project, strategy: this.strategy, patterns: this.patterns, cache, report}); }); + if (dedupeReport.hasErrors()) + return dedupeReport.exitCode(); + if (this.check) { return dedupedPackageCount ? 1 : 0; } else { @@ -266,8 +269,6 @@ export async function dedupe({strategy, project, patterns, cache, report}: Dedup ) ); - const prettyStrategy = configuration.format(strategy, FormatType.CODE); - let packages: string; switch (dedupedPackageCount) { case 0: { @@ -283,7 +284,8 @@ export async function dedupe({strategy, project, patterns, cache, report}: Dedup } } - report.reportInfo(MessageName.UNNAMED, `${packages} can be deduped using the strategy ${prettyStrategy}`); + const prettyStrategy = configuration.format(strategy, FormatType.CODE); + report.reportInfo(MessageName.UNNAMED, `${packages} can be deduped using the ${prettyStrategy} strategy`); return dedupedPackageCount; }); From 925112a7223ac31dcc55ac365a8324fbcd9517e1 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Wed, 19 Aug 2020 17:37:20 +0300 Subject: [PATCH 07/24] refactor: more refactoring + improved docs --- .../sources/commands/dedupe.ts | 45 ++++++++++++++++--- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index da40e780d545..52c578dec447 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -2,6 +2,10 @@ * Prior work: * - https://github.com/atlassian/yarn-deduplicate * - https://github.com/eps1lon/yarn-plugin-deduplicate + * + * Goals of the `dedupe` command: + * - the deduplication algorithms shouldn't depend on semver; they should instead use the resolver candidate system + * - the deduplication should happen concurrently */ import {BaseCommand} from '@yarnpkg/cli'; @@ -27,6 +31,13 @@ export type DedupeAlgorithm = (project: Project, patterns: Array, opts: }) => Promise>; export enum Strategy { + /** + * This strategy dedupes a locator to the best candidate already installed in the project. + * + * Because of this, it's guaranteed that: + * - it never takes more than a single pass to dedupe all dependencies + * - dependencies are never downgraded + */ HIGHEST = `highest`, } @@ -116,7 +127,7 @@ export default class DedupeCommand extends BaseCommand { @Command.String(`-s,--strategy`) strategy: Strategy = Strategy.HIGHEST; - @Command.Boolean(`--check`) + @Command.Boolean(`-c,--check`) check: boolean = false; @Command.Boolean(`--json`) @@ -138,22 +149,42 @@ export default class DedupeCommand extends BaseCommand { details: ` Duplicates are defined as descriptors with overlapping ranges being resolved and locked to different locators. They are a natural consequence of Yarn's deterministic installs, but they can sometimes pile up and unnecessarily increase the size of your project. - This command deduplicates dependencies in the current project by reusing (where possible) the locators with the highest versions. This means that dependencies can only be upgraded, never downgraded. + This command dedupes dependencies in the current project using different strategies (a single one is implemented at the moment): - **Note:** Although it never produces a wrong dependency tree, this command should be used with caution, as it modifies the dependency tree, which can sometimes cause problems when packages specify wrong dependency ranges. It is recommended to also review the changes manually. + - \`highest\`: Reuses (where possible) the locators with the highest versions. This means that dependencies can only be upgraded, never downgraded. It's also guaranteed that it never takes more than a single pass to dedupe the entire dependency tree. - If set, the \`--check\` flag will only report the found duplicates, without persisting the modified dependency tree. + **Note:** Even though it never produces a wrong dependency tree, this command should be used with caution, as it modifies the dependency tree, which can sometimes cause problems when packages specify wrong dependency ranges. It is recommended to also review the changes manually. + + If set, the \`-c,--check\` flag will only report the found duplicates, without persisting the modified dependency tree. This command accepts glob patterns as arguments (if valid Idents and supported by [micromatch](https://github.com/micromatch/micromatch)). Make sure to escape the patterns, to prevent your own shell from trying to expand them. + + ### In-depth explanation: + + > Note: The examples will use lockfiles trimmed-down to only contain the information needed to understand why this command is needed. + + Yarn doesn't deduplicate dependencies by default, otherwise installs wouldn't be deterministic and the lockfile would be useless. What it actually does is that it tries to not duplicate dependencies in the first place. + + **Example:** + + Running \`yarn add foo@*\` with the following lockfile ( + \`\`\`yml + foo@^2.3.4: + resolution: foo@2.3.4 + \`\`\` + ) will cause yarn to reuse \`foo@2.3.4\`, even if the latest \`foo\` is actually \`foo@2.10.14\`, thus preventing unnecessary duplication. `, examples: [[ - `Deduplicate all packages`, + `Dedupe all packages`, `$0 dedupe`, ], [ - `Deduplicate a specific package`, + `Dedupe all packages using a specific strategy`, + `$0 dedupe --strategy highest`, + ], [ + `Dedupe a specific package`, `$0 dedupe lodash`, ], [ - `Deduplicate all packages with the \`@babel\` scope`, + `Dedupe all packages with the \`@babel/*\` scope`, `$0 dedupe '@babel/*'`, ], [ `Check for duplicates (can be used as a CI step)`, From ebfd0c184ef78356455df050783af5da42794b27 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Wed, 19 Aug 2020 17:55:57 +0300 Subject: [PATCH 08/24] docs: better dedupe docs --- .../plugin-essentials/sources/commands/dedupe.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index 52c578dec447..8f686383ca94 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -149,7 +149,7 @@ export default class DedupeCommand extends BaseCommand { details: ` Duplicates are defined as descriptors with overlapping ranges being resolved and locked to different locators. They are a natural consequence of Yarn's deterministic installs, but they can sometimes pile up and unnecessarily increase the size of your project. - This command dedupes dependencies in the current project using different strategies (a single one is implemented at the moment): + This command dedupes dependencies in the current project using different strategies (only one is implemented at the moment): - \`highest\`: Reuses (where possible) the locators with the highest versions. This means that dependencies can only be upgraded, never downgraded. It's also guaranteed that it never takes more than a single pass to dedupe the entire dependency tree. @@ -161,18 +161,13 @@ export default class DedupeCommand extends BaseCommand { ### In-depth explanation: - > Note: The examples will use lockfiles trimmed-down to only contain the information needed to understand why this command is needed. - Yarn doesn't deduplicate dependencies by default, otherwise installs wouldn't be deterministic and the lockfile would be useless. What it actually does is that it tries to not duplicate dependencies in the first place. - **Example:** + **Example:** If \`foo@^2.3.4\` (a dependency of a dependency) has already been resolved to \`foo@2.3.4\`, running \`yarn add foo@*\`will cause Yarn to reuse \`foo@2.3.4\`, even if the latest \`foo\` is actually \`foo@2.10.14\`, thus preventing unnecessary duplication. + + Duplication happens when Yarn can't unlock dependencies that have already been locked inside the lockfile. - Running \`yarn add foo@*\` with the following lockfile ( - \`\`\`yml - foo@^2.3.4: - resolution: foo@2.3.4 - \`\`\` - ) will cause yarn to reuse \`foo@2.3.4\`, even if the latest \`foo\` is actually \`foo@2.10.14\`, thus preventing unnecessary duplication. + **Example:** If \`foo@^2.3.4\` (a dependency of a dependency) has already been resolved to \`foo@2.3.4\`, running \`yarn add foo@2.10.14\` will cause Yarn to install \`foo@2.10.14\` because the existing resolution doesn't satisfy the range \`2.10.14\`. This behavior can lead to (sometimes) unwanted duplication, since now the lockfile contains 2 separate resolutions for the 2 \`foo\` descriptors, even though they have overlapping ranges, which means that the lockfile can be simplified so that both descriptors resolve to \`foo@2.10.14\`. `, examples: [[ `Dedupe all packages`, From e8df62bc46abdb83cf1a28949f1b442a0990f1bd Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Wed, 19 Aug 2020 19:23:24 +0300 Subject: [PATCH 09/24] test: add tests --- .../pkg-tests-core/sources/utils/exec.ts | 2 +- .../packages/one-range-dep-too-1.0.0/index.js | 10 ++ .../one-range-dep-too-1.0.0/package.json | 7 + .../packages/two-range-deps-1.0.0/index.js | 10 ++ .../two-range-deps-1.0.0/package.json | 8 + .../sources/commands/dedupe.test.ts | 159 ++++++++++++++++++ 6 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 packages/acceptance-tests/pkg-tests-fixtures/packages/one-range-dep-too-1.0.0/index.js create mode 100644 packages/acceptance-tests/pkg-tests-fixtures/packages/one-range-dep-too-1.0.0/package.json create mode 100644 packages/acceptance-tests/pkg-tests-fixtures/packages/two-range-deps-1.0.0/index.js create mode 100644 packages/acceptance-tests/pkg-tests-fixtures/packages/two-range-deps-1.0.0/package.json create mode 100644 packages/acceptance-tests/pkg-tests-specs/sources/commands/dedupe.test.ts diff --git a/packages/acceptance-tests/pkg-tests-core/sources/utils/exec.ts b/packages/acceptance-tests/pkg-tests-core/sources/utils/exec.ts index 5543ae9d1ef0..ade8f5c48fc3 100644 --- a/packages/acceptance-tests/pkg-tests-core/sources/utils/exec.ts +++ b/packages/acceptance-tests/pkg-tests-core/sources/utils/exec.ts @@ -10,7 +10,7 @@ export type ExecResult = { stdout: string; stderr: string; code: number; -} | Error & { +} | cp.ExecException & { stdout: string; stderr: string; }; diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/one-range-dep-too-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/one-range-dep-too-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/one-range-dep-too-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/one-range-dep-too-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/one-range-dep-too-1.0.0/package.json new file mode 100644 index 000000000000..4df762811812 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/one-range-dep-too-1.0.0/package.json @@ -0,0 +1,7 @@ +{ + "name": "one-range-dep-too", + "version": "1.0.0", + "dependencies": { + "no-deps": "^1.0.0" + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/two-range-deps-1.0.0/index.js b/packages/acceptance-tests/pkg-tests-fixtures/packages/two-range-deps-1.0.0/index.js new file mode 100644 index 000000000000..a6bf8f586524 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/two-range-deps-1.0.0/index.js @@ -0,0 +1,10 @@ +/* @flow */ + +module.exports = require(`./package.json`); + +for (const key of [`dependencies`, `devDependencies`, `peerDependencies`]) { + for (const dep of Object.keys(module.exports[key] || {})) { + // $FlowFixMe The whole point of this file is to be dynamic + module.exports[key][dep] = require(dep); + } +} diff --git a/packages/acceptance-tests/pkg-tests-fixtures/packages/two-range-deps-1.0.0/package.json b/packages/acceptance-tests/pkg-tests-fixtures/packages/two-range-deps-1.0.0/package.json new file mode 100644 index 000000000000..a2279cdc19b3 --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-fixtures/packages/two-range-deps-1.0.0/package.json @@ -0,0 +1,8 @@ +{ + "name": "two-range-deps", + "version": "1.0.0", + "dependencies": { + "no-deps": "^1.0.0", + "@types/is-number": ">=1.0.0" + } +} diff --git a/packages/acceptance-tests/pkg-tests-specs/sources/commands/dedupe.test.ts b/packages/acceptance-tests/pkg-tests-specs/sources/commands/dedupe.test.ts new file mode 100644 index 000000000000..04024d57887a --- /dev/null +++ b/packages/acceptance-tests/pkg-tests-specs/sources/commands/dedupe.test.ts @@ -0,0 +1,159 @@ +import {tests} from 'pkg-tests-core'; + +const {setPackageWhitelist} = tests; + +describe(`Commands`, () => { + describe(`dedupe`, () => { + it( + `should include a footer`, + makeTemporaryEnv({}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([ + [`no-deps`, new Set([`1.0.0`])], + [`@types/is-number`, new Set([`1.0.0`])], + ]), async () => { + await run(`add`, `two-range-deps`); + }); + + await run(`add`, `no-deps@1.1.0`, `@types/is-number@2.0.0`); + + await expect(run(`dedupe`, `--check`)).rejects.toMatchObject({ + stdout: expect.stringContaining(`2 packages can be deduped using the highest strategy`), + }); + }) + ); + + describe(`strategies`, () => { + describe(`highest`, () => { + it( + `should dedupe dependencies`, + makeTemporaryEnv({}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`); + }); + + await run(`add`, `no-deps@1.1.0`); + + await run(`dedupe`); + + await expect(run(`dedupe`, `--check`)).resolves.toMatchObject({ + code: 0, + }); + + await expect(source(`require('no-deps')`)).resolves.toMatchObject({ + version: `1.1.0`, + }); + await expect(source(`require('one-range-dep')`)).resolves.toMatchObject({ + dependencies: { + [`no-deps`]: { + version: `1.1.0`, + }, + }, + }); + }) + ); + + it( + `should dedupe dependencies to the highest possible version`, + makeTemporaryEnv({}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`, `one-range-dep-too`); + }); + + await run(`add`, `no-deps@1.1.0`); + + await run(`dedupe`); + + await expect(source(`require('no-deps')`)).resolves.toMatchObject({ + version: `1.1.0`, + }); + await expect(source(`require('one-range-dep')`)).resolves.toMatchObject({ + dependencies: { + [`no-deps`]: { + version: `1.1.0`, + }, + }, + }); + await expect(source(`require('one-range-dep-too')`)).resolves.toMatchObject({ + dependencies: { + [`no-deps`]: { + version: `1.1.0`, + }, + }, + }); + }) + ); + }); + }); + + describe(`flags`, () => { + describe(`-c,--check`, () => { + it( + `should reject with error code 1 when there are duplicates`, + makeTemporaryEnv({}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`); + }); + + await run(`add`, `no-deps@1.1.0`); + + await expect(run(`dedupe`, `--check`)).rejects.toMatchObject({ + code: 1, + }); + }) + ); + + it( + `should resolve with error code 0 when there are no duplicates`, + makeTemporaryEnv({}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`); + }); + + await run(`add`, `no-deps@2.0.0`); + + await expect(run(`dedupe`, `--check`)).resolves.toMatchObject({ + code: 0, + }); + }) + ); + }); + + test( + `--json`, + makeTemporaryEnv({}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`); + }); + + await run(`add`, `no-deps@1.1.0`); + + // We also use the check flag so that the stdout doesn't include the install report + await run(`dedupe`, `--json`, `--check`).catch(({stdout}) => { + expect(JSON.parse(stdout.trim())).toMatchObject({ + descriptor: `no-deps@npm:^1.0.0`, + currentResolution: `no-deps@npm:1.0.0`, + updatedResolution: `no-deps@npm:1.1.0`, + }); + }); + + expect.assertions(1); + }) + ); + + test( + `-s,--strategy`, + makeTemporaryEnv({}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`); + }); + + await run(`add`, `no-deps@1.1.0`); + + await expect(run(`dedupe`, `--check`, `--strategy`, `highest`)).rejects.toMatchObject({ + code: 1, + }); + }) + ); + }); + }); +}); From 658055c7f4cfb77be326417e6a3e3b4689b8d82a Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Wed, 19 Aug 2020 19:47:17 +0300 Subject: [PATCH 10/24] test: add tests for selective dedupe --- .../sources/commands/dedupe.test.ts | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/packages/acceptance-tests/pkg-tests-specs/sources/commands/dedupe.test.ts b/packages/acceptance-tests/pkg-tests-specs/sources/commands/dedupe.test.ts index 04024d57887a..b5c52648b957 100644 --- a/packages/acceptance-tests/pkg-tests-specs/sources/commands/dedupe.test.ts +++ b/packages/acceptance-tests/pkg-tests-specs/sources/commands/dedupe.test.ts @@ -85,6 +85,116 @@ describe(`Commands`, () => { }); }); + describe(`patterns`, () => { + it( + `should support selective dedupe (ident)`, + makeTemporaryEnv({}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([ + [`no-deps`, new Set([`1.0.0`])], + [`@types/is-number`, new Set([`1.0.0`])], + ]), async () => { + await run(`add`, `two-range-deps`); + }); + + await run(`add`, `no-deps@1.1.0`, `@types/is-number@2.0.0`); + + await run(`dedupe`, `no-deps`); + + await expect(source(`require('two-range-deps')`)).resolves.toMatchObject({ + dependencies: { + [`no-deps`]: { + version: `1.1.0`, + }, + [`@types/is-number`]: { + version: `1.0.0`, + }, + }, + }); + }) + ); + + it( + `should support selective dedupe (scoped ident)`, + makeTemporaryEnv({}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([ + [`no-deps`, new Set([`1.0.0`])], + [`@types/is-number`, new Set([`1.0.0`])], + ]), async () => { + await run(`add`, `two-range-deps`); + }); + + await run(`add`, `no-deps@1.1.0`, `@types/is-number@2.0.0`); + + await run(`dedupe`, `@types/is-number`); + + await expect(source(`require('two-range-deps')`)).resolves.toMatchObject({ + dependencies: { + [`no-deps`]: { + version: `1.0.0`, + }, + [`@types/is-number`]: { + version: `2.0.0`, + }, + }, + }); + }) + ); + + it( + `should support selective dedupe (ident glob)`, + makeTemporaryEnv({}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([ + [`no-deps`, new Set([`1.0.0`])], + [`@types/is-number`, new Set([`1.0.0`])], + ]), async () => { + await run(`add`, `two-range-deps`); + }); + + await run(`add`, `no-deps@1.1.0`, `@types/is-number@2.0.0`); + + await run(`dedupe`, `no-*`); + + await expect(source(`require('two-range-deps')`)).resolves.toMatchObject({ + dependencies: { + [`no-deps`]: { + version: `1.1.0`, + }, + [`@types/is-number`]: { + version: `1.0.0`, + }, + }, + }); + }) + ); + + it( + `should support selective dedupe (scoped ident glob)`, + makeTemporaryEnv({}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([ + [`no-deps`, new Set([`1.0.0`])], + [`@types/is-number`, new Set([`1.0.0`])], + ]), async () => { + await run(`add`, `two-range-deps`); + }); + + await run(`add`, `no-deps@1.1.0`, `@types/is-number@2.0.0`); + + await run(`dedupe`, `@types/*`); + + await expect(source(`require('two-range-deps')`)).resolves.toMatchObject({ + dependencies: { + [`no-deps`]: { + version: `1.0.0`, + }, + [`@types/is-number`]: { + version: `2.0.0`, + }, + }, + }); + }) + ); + }); + describe(`flags`, () => { describe(`-c,--check`, () => { it( From bb60e25a115e2380e0da5a753c5aa79142706290 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Fri, 21 Aug 2020 00:18:56 +0300 Subject: [PATCH 11/24] refactor: refactor everything yet again --- .yarn/versions/e9b298c7.yml | 12 +++- .../sources/commands/dedupe.ts | 22 ++++-- packages/plugin-exec/sources/ExecResolver.ts | 46 +++++++----- packages/plugin-file/sources/FileResolver.ts | 70 +++++++++++-------- .../sources/TarballFileResolver.ts | 34 ++++++--- packages/plugin-git/sources/GitResolver.ts | 28 ++++++-- .../sources/TarballHttpResolver.ts | 18 +++-- packages/plugin-link/sources/LinkResolver.ts | 30 +++++--- .../plugin-link/sources/RawLinkResolver.ts | 30 +++++--- .../plugin-npm/sources/NpmRemapResolver.ts | 6 ++ .../plugin-npm/sources/NpmSemverResolver.ts | 12 ++++ packages/plugin-npm/sources/NpmTagResolver.ts | 17 +++-- .../plugin-patch/sources/PatchResolver.ts | 30 +++++--- .../sources/LegacyMigrationResolver.ts | 15 +++- .../yarnpkg-core/sources/LockfileResolver.ts | 18 ++++- .../yarnpkg-core/sources/MultiResolver.ts | 6 ++ .../yarnpkg-core/sources/ProtocolResolver.ts | 4 ++ packages/yarnpkg-core/sources/Resolver.ts | 23 ++++++ .../sources/RunInstallPleaseResolver.ts | 4 ++ .../yarnpkg-core/sources/VirtualResolver.ts | 10 ++- .../yarnpkg-core/sources/WorkspaceResolver.ts | 11 ++- 21 files changed, 337 insertions(+), 109 deletions(-) diff --git a/.yarn/versions/e9b298c7.yml b/.yarn/versions/e9b298c7.yml index e9b059fffe3b..40475073c400 100644 --- a/.yarn/versions/e9b298c7.yml +++ b/.yarn/versions/e9b298c7.yml @@ -1,22 +1,30 @@ releases: "@yarnpkg/cli": prerelease + "@yarnpkg/core": minor "@yarnpkg/plugin-essentials": minor + "@yarnpkg/plugin-exec": patch + "@yarnpkg/plugin-file": patch + "@yarnpkg/plugin-git": patch + "@yarnpkg/plugin-http": patch + "@yarnpkg/plugin-link": patch + "@yarnpkg/plugin-npm": patch + "@yarnpkg/plugin-patch": patch declined: - "@yarnpkg/plugin-compat" - "@yarnpkg/plugin-constraints" - "@yarnpkg/plugin-dlx" + - "@yarnpkg/plugin-github" - "@yarnpkg/plugin-init" - "@yarnpkg/plugin-interactive-tools" - "@yarnpkg/plugin-node-modules" - "@yarnpkg/plugin-npm-cli" - "@yarnpkg/plugin-pack" - - "@yarnpkg/plugin-patch" - "@yarnpkg/plugin-pnp" - "@yarnpkg/plugin-stage" - "@yarnpkg/plugin-typescript" - "@yarnpkg/plugin-version" - "@yarnpkg/plugin-workspace-tools" - "@yarnpkg/builder" - - "@yarnpkg/core" - "@yarnpkg/doctor" + - "@yarnpkg/pnpify" diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index 8f686383ca94..557eff8defe2 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -83,24 +83,32 @@ export const DEDUPE_ALGORITHMS: Record = { if (locators.size === 1) return dedupeSkip; + const references = [...locators].map(locatorHash => { + const pkg = project.originalPackages.get(locatorHash); + if (typeof pkg === `undefined`) + throw new Error(`Assertion failed: The package (${locatorHash}) should have been registered`); + + return pkg.reference; + }); + const resolutionDependencies = resolver.getResolutionDependencies(descriptor, resolveOptions); const dependencies = new Map( - resolutionDependencies.map(dependency => { - const resolution = project.storedResolutions.get(dependency.descriptorHash); + resolutionDependencies.map(({descriptorHash}) => { + const resolution = project.storedResolutions.get(descriptorHash); if (typeof resolution === `undefined`) - throw new Error(`Assertion failed: The resolution (${structUtils.prettyDescriptor(project.configuration, dependency)}) should have been registered`); + throw new Error(`Assertion failed: The resolution (${descriptorHash}) should have been registered`); - const pkg = project.storedPackages.get(resolution); + const pkg = project.originalPackages.get(resolution); if (typeof pkg === `undefined`) throw new Error(`Assertion failed: The package (${resolution}) should have been registered`); - return [dependency.descriptorHash, pkg] as const; + return [descriptorHash, pkg] as const; }) ); - const candidates = await resolver.getCandidates(descriptor, dependencies, resolveOptions); + const candidates = await resolver.getSatisfying(descriptor, references, dependencies, resolveOptions); - const bestCandidate = candidates.find(({locatorHash}) => locators.has(locatorHash)); + const bestCandidate = candidates[0]; if (typeof bestCandidate === `undefined`) return dedupeSkip; diff --git a/packages/plugin-exec/sources/ExecResolver.ts b/packages/plugin-exec/sources/ExecResolver.ts index 29ecd6be4161..ff9d96394251 100644 --- a/packages/plugin-exec/sources/ExecResolver.ts +++ b/packages/plugin-exec/sources/ExecResolver.ts @@ -39,25 +39,17 @@ export class ExecResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: Map, opts: ResolveOptions) { - if (!opts.fetchOptions) - throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); + const locator = await this.getCandidateForDescriptor(descriptor, opts); - const {path, parentLocator} = execUtils.parseSpec(descriptor.range); - - if (parentLocator === null) - throw new Error(`Assertion failed: The descriptor should have been bound`); + return [locator]; + } - const generatorFile = await execUtils.loadGeneratorFile(structUtils.makeRange({ - protocol: PROTOCOL, - source: path, - selector: path, - params: { - locator: structUtils.stringifyLocator(parentLocator), - }, - }), PROTOCOL, opts.fetchOptions); - const generatorHash = hashUtils.makeHash(`${CACHE_VERSION}`, generatorFile).slice(0, 6); + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const {reference: execReference} = await this.getCandidateForDescriptor(descriptor, opts); - return [execUtils.makeLocator(descriptor, {parentLocator, path, generatorHash, protocol: PROTOCOL})]; + return references + .filter(reference => reference === execReference) + .map(reference => structUtils.makeLocator(descriptor, reference)); } async resolve(locator: Locator, opts: ResolveOptions) { @@ -87,4 +79,26 @@ export class ExecResolver implements Resolver { bin: manifest.bin, }; } + + private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { + if (!opts.fetchOptions) + throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); + + const {path, parentLocator} = execUtils.parseSpec(descriptor.range); + + if (parentLocator === null) + throw new Error(`Assertion failed: The descriptor should have been bound`); + + const generatorFile = await execUtils.loadGeneratorFile(structUtils.makeRange({ + protocol: PROTOCOL, + source: path, + selector: path, + params: { + locator: structUtils.stringifyLocator(parentLocator), + }, + }), PROTOCOL, opts.fetchOptions); + const generatorHash = hashUtils.makeHash(`${CACHE_VERSION}`, generatorFile).slice(0, 6); + + return execUtils.makeLocator(descriptor, {parentLocator, path, generatorHash, protocol: PROTOCOL}); + } } diff --git a/packages/plugin-file/sources/FileResolver.ts b/packages/plugin-file/sources/FileResolver.ts index 8c97ba9a8326..875f4e1b4066 100644 --- a/packages/plugin-file/sources/FileResolver.ts +++ b/packages/plugin-file/sources/FileResolver.ts @@ -1,10 +1,10 @@ -import {miscUtils, structUtils, hashUtils} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; -import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; +import {miscUtils, structUtils, hashUtils, DescriptorHash, Package} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; +import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; -import {FILE_REGEXP, PROTOCOL} from './constants'; -import * as fileUtils from './fileUtils'; +import {FILE_REGEXP, PROTOCOL} from './constants'; +import * as fileUtils from './fileUtils'; // We use this for the folders to be regenerated without bumping the whole cache const CACHE_VERSION = 1; @@ -45,31 +45,17 @@ export class FileResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) { - if (!opts.fetchOptions) - throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); - - const {path, parentLocator} = fileUtils.parseSpec(descriptor.range); - - if (parentLocator === null) - throw new Error(`Assertion failed: The descriptor should have been bound`); + const locator = await this.getCandidateForDescriptor(descriptor, opts); - const archiveBuffer = await fileUtils.makeBufferFromLocator( - structUtils.makeLocator(descriptor, - structUtils.makeRange({ - protocol: PROTOCOL, - source: path, - selector: path, - params: { - locator: structUtils.stringifyLocator(parentLocator), - }, - }) - ), - {protocol: PROTOCOL, fetchOptions: opts.fetchOptions} - ); + return [locator]; + } - const folderHash = hashUtils.makeHash(`${CACHE_VERSION}`, archiveBuffer).slice(0, 6); + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const {reference: fileReference} = await this.getCandidateForDescriptor(descriptor, opts); - return [fileUtils.makeLocator(descriptor, {parentLocator, path, folderHash, protocol: PROTOCOL})]; + return references + .filter(reference => reference === fileReference) + .map(reference => structUtils.makeLocator(descriptor, reference)); } async resolve(locator: Locator, opts: ResolveOptions) { @@ -99,4 +85,32 @@ export class FileResolver implements Resolver { bin: manifest.bin, }; } + + private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { + if (!opts.fetchOptions) + throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); + + const {path, parentLocator} = fileUtils.parseSpec(descriptor.range); + + if (parentLocator === null) + throw new Error(`Assertion failed: The descriptor should have been bound`); + + const archiveBuffer = await fileUtils.makeBufferFromLocator( + structUtils.makeLocator(descriptor, + structUtils.makeRange({ + protocol: PROTOCOL, + source: path, + selector: path, + params: { + locator: structUtils.stringifyLocator(parentLocator), + }, + }) + ), + {protocol: PROTOCOL, fetchOptions: opts.fetchOptions} + ); + + const folderHash = hashUtils.makeHash(`${CACHE_VERSION}`, archiveBuffer).slice(0, 6); + + return fileUtils.makeLocator(descriptor, {parentLocator, path, folderHash, protocol: PROTOCOL}); + } } diff --git a/packages/plugin-file/sources/TarballFileResolver.ts b/packages/plugin-file/sources/TarballFileResolver.ts index 03b2e8a6e650..48956dcbb1dd 100644 --- a/packages/plugin-file/sources/TarballFileResolver.ts +++ b/packages/plugin-file/sources/TarballFileResolver.ts @@ -1,10 +1,10 @@ -import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; -import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {miscUtils, structUtils} from '@yarnpkg/core'; -import {npath} from '@yarnpkg/fslib'; +import {Resolver, ResolveOptions, MinimalResolveOptions, DescriptorHash, Package} from '@yarnpkg/core'; +import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {miscUtils, structUtils} from '@yarnpkg/core'; +import {npath} from '@yarnpkg/fslib'; -import {FILE_REGEXP, TARBALL_REGEXP, PROTOCOL} from './constants'; +import {FILE_REGEXP, TARBALL_REGEXP, PROTOCOL} from './constants'; export class TarballFileResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -48,12 +48,17 @@ export class TarballFileResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) { - let path = descriptor.range; + const locator = await this.getCandidateForDescriptor(descriptor, opts); - if (path.startsWith(PROTOCOL)) - path = path.slice(PROTOCOL.length); + return [locator]; + } + + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const {reference: tarballFileReference} = await this.getCandidateForDescriptor(descriptor, opts); - return [structUtils.makeLocator(descriptor, `${PROTOCOL}${npath.toPortablePath(path)}`)]; + return references + .filter(reference => reference === tarballFileReference) + .map(reference => structUtils.makeLocator(descriptor, reference)); } async resolve(locator: Locator, opts: ResolveOptions) { @@ -83,4 +88,13 @@ export class TarballFileResolver implements Resolver { bin: manifest.bin, }; } + + private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { + let path = descriptor.range; + + if (path.startsWith(PROTOCOL)) + path = path.slice(PROTOCOL.length); + + return structUtils.makeLocator(descriptor, `${PROTOCOL}${npath.toPortablePath(path)}`); + } } diff --git a/packages/plugin-git/sources/GitResolver.ts b/packages/plugin-git/sources/GitResolver.ts index 8ff545a1e766..9dda0ca7c504 100644 --- a/packages/plugin-git/sources/GitResolver.ts +++ b/packages/plugin-git/sources/GitResolver.ts @@ -1,9 +1,9 @@ -import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; -import {miscUtils, structUtils} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; +import {Resolver, ResolveOptions, MinimalResolveOptions, DescriptorHash, Package} from '@yarnpkg/core'; +import {miscUtils, structUtils} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; -import * as gitUtils from './gitUtils'; +import * as gitUtils from './gitUtils'; export class GitResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -27,12 +27,19 @@ export class GitResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) { - const reference = await gitUtils.resolveUrl(descriptor.range, opts.project.configuration); - const locator = structUtils.makeLocator(descriptor, reference); + const locator = await this.getCandidateForDescriptor(descriptor, opts); return [locator]; } + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const {reference: gitReference} = await this.getCandidateForDescriptor(descriptor, opts); + + return references + .filter(reference => reference === gitReference) + .map(reference => structUtils.makeLocator(descriptor, reference)); + } + async resolve(locator: Locator, opts: ResolveOptions) { if (!opts.fetchOptions) throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); @@ -60,4 +67,11 @@ export class GitResolver implements Resolver { bin: manifest.bin, }; } + + private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { + const reference = await gitUtils.resolveUrl(descriptor.range, opts.project.configuration); + const locator = structUtils.makeLocator(descriptor, reference); + + return locator; + } } diff --git a/packages/plugin-http/sources/TarballHttpResolver.ts b/packages/plugin-http/sources/TarballHttpResolver.ts index 026282c1da78..0d6672572099 100644 --- a/packages/plugin-http/sources/TarballHttpResolver.ts +++ b/packages/plugin-http/sources/TarballHttpResolver.ts @@ -1,9 +1,9 @@ -import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; -import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {miscUtils, structUtils} from '@yarnpkg/core'; +import {Resolver, ResolveOptions, MinimalResolveOptions, DescriptorHash, Package} from '@yarnpkg/core'; +import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {miscUtils, structUtils} from '@yarnpkg/core'; -import {PROTOCOL_REGEXP, TARBALL_REGEXP} from './constants'; +import {PROTOCOL_REGEXP, TARBALL_REGEXP} from './constants'; export class TarballHttpResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -42,6 +42,14 @@ export class TarballHttpResolver implements Resolver { return [structUtils.convertDescriptorToLocator(descriptor)]; } + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const tarballReference = structUtils.convertDescriptorToLocator(descriptor).reference; + + return references + .filter(reference => reference === tarballReference) + .map(reference => structUtils.makeLocator(descriptor, reference)); + } + async resolve(locator: Locator, opts: ResolveOptions) { if (!opts.fetchOptions) throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); diff --git a/packages/plugin-link/sources/LinkResolver.ts b/packages/plugin-link/sources/LinkResolver.ts index ebfef27c47c7..96a219b0cf8b 100644 --- a/packages/plugin-link/sources/LinkResolver.ts +++ b/packages/plugin-link/sources/LinkResolver.ts @@ -1,10 +1,10 @@ -import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; -import {Descriptor, Locator, Manifest, Package} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {miscUtils, structUtils} from '@yarnpkg/core'; -import {npath} from '@yarnpkg/fslib'; +import {Resolver, ResolveOptions, MinimalResolveOptions, DescriptorHash} from '@yarnpkg/core'; +import {Descriptor, Locator, Manifest, Package} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {miscUtils, structUtils} from '@yarnpkg/core'; +import {npath} from '@yarnpkg/fslib'; -import {LINK_PROTOCOL} from './constants'; +import {LINK_PROTOCOL} from './constants'; export class LinkResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -36,9 +36,17 @@ export class LinkResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) { - const path = descriptor.range.slice(LINK_PROTOCOL.length); + const locator = await this.getCandidateForDescriptor(descriptor, opts); + + return [locator]; + } + + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const {reference: linkReference} = await this.getCandidateForDescriptor(descriptor, opts); - return [structUtils.makeLocator(descriptor, `${LINK_PROTOCOL}${npath.toPortablePath(path)}`)]; + return references + .filter(reference => reference === linkReference) + .map(reference => structUtils.makeLocator(descriptor, reference)); } async resolve(locator: Locator, opts: ResolveOptions): Promise { @@ -68,4 +76,10 @@ export class LinkResolver implements Resolver { bin: manifest.bin, }; } + + private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { + const path = descriptor.range.slice(LINK_PROTOCOL.length); + + return structUtils.makeLocator(descriptor, `${LINK_PROTOCOL}${npath.toPortablePath(path)}`); + } } diff --git a/packages/plugin-link/sources/RawLinkResolver.ts b/packages/plugin-link/sources/RawLinkResolver.ts index bdf9619ce4ad..59cfbe7f260c 100644 --- a/packages/plugin-link/sources/RawLinkResolver.ts +++ b/packages/plugin-link/sources/RawLinkResolver.ts @@ -1,10 +1,10 @@ -import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; -import {Descriptor, Locator} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {structUtils} from '@yarnpkg/core'; -import {npath} from '@yarnpkg/fslib'; +import {Resolver, ResolveOptions, MinimalResolveOptions, DescriptorHash, Package} from '@yarnpkg/core'; +import {Descriptor, Locator} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {structUtils} from '@yarnpkg/core'; +import {npath} from '@yarnpkg/fslib'; -import {RAW_LINK_PROTOCOL} from './constants'; +import {RAW_LINK_PROTOCOL} from './constants'; export class RawLinkResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -36,9 +36,17 @@ export class RawLinkResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) { - const path = descriptor.range.slice(RAW_LINK_PROTOCOL.length); + const locator = await this.getCandidateForDescriptor(descriptor, opts); + + return [locator]; + } + + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const {reference: rawLinkReference} = await this.getCandidateForDescriptor(descriptor, opts); - return [structUtils.makeLocator(descriptor, `${RAW_LINK_PROTOCOL}${npath.toPortablePath(path)}`)]; + return references + .filter(reference => reference === rawLinkReference) + .map(reference => structUtils.makeLocator(descriptor, reference)); } async resolve(locator: Locator, opts: ResolveOptions) { @@ -59,4 +67,10 @@ export class RawLinkResolver implements Resolver { bin: new Map(), }; } + + private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { + const path = descriptor.range.slice(RAW_LINK_PROTOCOL.length); + + return structUtils.makeLocator(descriptor, `${RAW_LINK_PROTOCOL}${npath.toPortablePath(path)}`); + } } diff --git a/packages/plugin-npm/sources/NpmRemapResolver.ts b/packages/plugin-npm/sources/NpmRemapResolver.ts index b584c02f2e2b..2746db6b88c3 100644 --- a/packages/plugin-npm/sources/NpmRemapResolver.ts +++ b/packages/plugin-npm/sources/NpmRemapResolver.ts @@ -40,6 +40,12 @@ export class NpmRemapResolver implements Resolver { return await opts.resolver.getCandidates(nextDescriptor, dependencies, opts); } + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const nextDescriptor = structUtils.parseDescriptor(descriptor.range.slice(PROTOCOL.length), true); + + return opts.resolver.getSatisfying(nextDescriptor, references, dependencies, opts); + } + resolve(locator: Locator, opts: ResolveOptions): never { // Once transformed into locators, the descriptors are resolved by the NpmSemverResolver throw new Error(`Unreachable`); diff --git a/packages/plugin-npm/sources/NpmSemverResolver.ts b/packages/plugin-npm/sources/NpmSemverResolver.ts index 7a363401bd74..ea08024b39e0 100644 --- a/packages/plugin-npm/sources/NpmSemverResolver.ts +++ b/packages/plugin-npm/sources/NpmSemverResolver.ts @@ -82,6 +82,18 @@ export class NpmSemverResolver implements Resolver { }); } + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const range = descriptor.range.slice(PROTOCOL.length); + const versions = references + .map(reference => reference.slice(PROTOCOL.length)) + .filter(reference => semver.valid(reference)); + + return versions + .filter(version => semver.satisfies(version, range)) + .sort((a, b) => -semver.compare(a, b)) + .map(version => structUtils.makeLocator(descriptor, `${PROTOCOL}${version}`)); + } + async resolve(locator: Locator, opts: ResolveOptions) { const {selector} = structUtils.parseRange(locator.reference); diff --git a/packages/plugin-npm/sources/NpmTagResolver.ts b/packages/plugin-npm/sources/NpmTagResolver.ts index 7abb245b677e..bef62dfa682b 100644 --- a/packages/plugin-npm/sources/NpmTagResolver.ts +++ b/packages/plugin-npm/sources/NpmTagResolver.ts @@ -1,10 +1,10 @@ -import {ReportError, MessageName, Resolver, ResolveOptions, MinimalResolveOptions, TAG_REGEXP} from '@yarnpkg/core'; -import {structUtils} from '@yarnpkg/core'; -import {Descriptor, Locator, Package} from '@yarnpkg/core'; +import {ReportError, MessageName, Resolver, ResolveOptions, MinimalResolveOptions, TAG_REGEXP, DescriptorHash} from '@yarnpkg/core'; +import {structUtils} from '@yarnpkg/core'; +import {Descriptor, Locator, Package} from '@yarnpkg/core'; -import {NpmSemverFetcher} from './NpmSemverFetcher'; -import {PROTOCOL} from './constants'; -import * as npmHttpUtils from './npmHttpUtils'; +import {NpmSemverFetcher} from './NpmSemverFetcher'; +import {PROTOCOL} from './constants'; +import * as npmHttpUtils from './npmHttpUtils'; export class NpmTagResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -64,6 +64,11 @@ export class NpmTagResolver implements Resolver { } } + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + // We can't statically know if a tag resolves to a specific version without using the network + return []; + } + async resolve(locator: Locator, opts: ResolveOptions): Promise { // Once transformed into locators (through getCandidates), the tags are resolved by the NpmSemverResolver throw new Error(`Unreachable`); diff --git a/packages/plugin-patch/sources/PatchResolver.ts b/packages/plugin-patch/sources/PatchResolver.ts index 8afbf1dad167..79a702d1a1d0 100644 --- a/packages/plugin-patch/sources/PatchResolver.ts +++ b/packages/plugin-patch/sources/PatchResolver.ts @@ -46,6 +46,27 @@ export class PatchResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: Map, opts: ResolveOptions) { + const locator = await this.getCandidateForDescriptor(descriptor, dependencies, opts); + + return [locator]; + } + + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const {reference: patchReference} = await this.getCandidateForDescriptor(descriptor, dependencies, opts); + + return references + .filter(reference => reference === patchReference) + .map(reference => structUtils.makeLocator(descriptor, reference)); + } + + async resolve(locator: Locator, opts: ResolveOptions): Promise { + const {sourceLocator} = patchUtils.parseLocator(locator); + const sourcePkg = await opts.resolver.resolve(sourceLocator, opts); + + return {...sourcePkg, ...locator}; + } + + private async getCandidateForDescriptor(descriptor: Descriptor, dependencies: Map, opts: ResolveOptions) { if (!opts.fetchOptions) throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); @@ -58,13 +79,6 @@ export class PatchResolver implements Resolver { const patchHash = hashUtils.makeHash(`${CACHE_VERSION}`, ...patchFiles).slice(0, 6); - return [patchUtils.makeLocator(descriptor, {parentLocator, sourcePackage, patchPaths, patchHash})]; - } - - async resolve(locator: Locator, opts: ResolveOptions): Promise { - const {sourceLocator} = patchUtils.parseLocator(locator); - const sourcePkg = await opts.resolver.resolve(sourceLocator, opts); - - return {...sourcePkg, ...locator}; + return patchUtils.makeLocator(descriptor, {parentLocator, sourcePackage, patchPaths, patchHash}); } } diff --git a/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts b/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts index ec6d3aea6084..87f2015db5ed 100644 --- a/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts +++ b/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts @@ -7,7 +7,7 @@ import {Project} from './Project'; import {Report} from './Report'; import {Resolver, ResolveOptions, MinimalResolveOptions} from './Resolver'; import * as structUtils from './structUtils'; -import {DescriptorHash, Descriptor, Locator} from './types'; +import {DescriptorHash, Descriptor, Locator, Package} from './types'; const IMPORTED_PATTERNS: Array<[RegExp, (version: string, ...args: Array) => string]> = [ // These ones come from Git urls @@ -115,6 +115,19 @@ export class LegacyMigrationResolver implements Resolver { return [resolution]; } + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + if (!this.resolutions) + throw new Error(`Assertion failed: The resolution store should have been setup`); + + const resolution = this.resolutions.get(descriptor.descriptorHash); + if (!resolution) + throw new Error(`Assertion failed: The resolution should have been registered`); + + return references + .filter(reference => reference === resolution.reference) + .map(reference => structUtils.makeLocator(descriptor, reference)); + } + async resolve(locator: Locator, opts: ResolveOptions): Promise { throw new Error(`Assertion failed: This resolver doesn't support resolving locators to packages`); } diff --git a/packages/yarnpkg-core/sources/LockfileResolver.ts b/packages/yarnpkg-core/sources/LockfileResolver.ts index 216efb5c9ca3..e0ed90f8fe80 100644 --- a/packages/yarnpkg-core/sources/LockfileResolver.ts +++ b/packages/yarnpkg-core/sources/LockfileResolver.ts @@ -1,6 +1,6 @@ import {Resolver, ResolveOptions, MinimalResolveOptions} from './Resolver'; import * as structUtils from './structUtils'; -import {Descriptor, Locator} from './types'; +import {Descriptor, Locator, DescriptorHash, Package} from './types'; export class LockfileResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -51,6 +51,22 @@ export class LockfileResolver implements Resolver { return [pkg]; } + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const convertedPkg = opts.project.originalPackages.get(structUtils.convertDescriptorToLocator(descriptor).locatorHash); + + const resolution = opts.project.storedResolutions.get(descriptor.descriptorHash); + const resolutionPkg = typeof resolution !== `undefined` + ? opts.project.originalPackages.get(resolution) + : undefined; + + return references + .filter(reference => [ + convertedPkg?.reference, + resolutionPkg?.reference, + ].includes(reference)) + .map(reference => structUtils.makeLocator(descriptor, reference)); + } + async resolve(locator: Locator, opts: ResolveOptions) { const pkg = opts.project.originalPackages.get(locator.locatorHash); diff --git a/packages/yarnpkg-core/sources/MultiResolver.ts b/packages/yarnpkg-core/sources/MultiResolver.ts index f2d19e6e5bf1..411b7d568670 100644 --- a/packages/yarnpkg-core/sources/MultiResolver.ts +++ b/packages/yarnpkg-core/sources/MultiResolver.ts @@ -45,6 +45,12 @@ export class MultiResolver implements Resolver { return await resolver.getCandidates(descriptor, dependencies, opts); } + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const resolver = this.getResolverByDescriptor(descriptor, opts); + + return resolver.getSatisfying(descriptor, references, dependencies, opts); + } + async resolve(locator: Locator, opts: ResolveOptions) { const resolver = this.getResolverByLocator(locator, opts); diff --git a/packages/yarnpkg-core/sources/ProtocolResolver.ts b/packages/yarnpkg-core/sources/ProtocolResolver.ts index 788d16526497..e2560ac96822 100644 --- a/packages/yarnpkg-core/sources/ProtocolResolver.ts +++ b/packages/yarnpkg-core/sources/ProtocolResolver.ts @@ -43,6 +43,10 @@ export class ProtocolResolver implements Resolver { return await opts.resolver.getCandidates(this.forwardDescriptor(descriptor, opts), dependencies, opts); } + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + return await opts.resolver.getSatisfying(this.forwardDescriptor(descriptor, opts), references, dependencies, opts); + } + async resolve(locator: Locator, opts: ResolveOptions) { const pkg = await opts.resolver.resolve(this.forwardLocator(locator, opts), opts); diff --git a/packages/yarnpkg-core/sources/Resolver.ts b/packages/yarnpkg-core/sources/Resolver.ts index 26d32dd823d7..33ff68004582 100644 --- a/packages/yarnpkg-core/sources/Resolver.ts +++ b/packages/yarnpkg-core/sources/Resolver.ts @@ -122,6 +122,29 @@ export interface Resolver { */ getCandidates(descriptor: Descriptor, dependencies: Map, opts: ResolveOptions): Promise>; + /** + * This function will, given a descriptor and a list of locator references, + * find out which of the references potentially satisfy the descriptor. + * + * This function is different from `getCandidates`, as `getCandidates` will + * resolve the descriptor into a list of locators (potentially using the network), + * while `getSatisfying` will statically compute which known references potentially + * satisfy the target descriptor. + * + * Note that the parameter references aren't guaranteed to be supported by + * the resolver, so they'll probably need to be filtered beforehand. + * + * The returned array must be sorted in such a way that the preferred + * locators are first. This will cause the resolution algorithm to prioritize + * them if possible (it doesn't guarantee that they'll end up being used). + * + * @param descriptor The target descriptor. + * @param references The candidate references. + * @param dependencies The resolution dependencies and their resolutions. + * @param opts The resolution options. + */ + getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions): Promise>; + /** * This function will, given a locator, return the full package definition * for the package pointed at. diff --git a/packages/yarnpkg-core/sources/RunInstallPleaseResolver.ts b/packages/yarnpkg-core/sources/RunInstallPleaseResolver.ts index 72f7f45c5823..140ac1a477a5 100644 --- a/packages/yarnpkg-core/sources/RunInstallPleaseResolver.ts +++ b/packages/yarnpkg-core/sources/RunInstallPleaseResolver.ts @@ -34,6 +34,10 @@ export class RunInstallPleaseResolver implements Resolver { throw new ReportError(MessageName.MISSING_LOCKFILE_ENTRY, `This package doesn't seem to be present in your lockfile; try to make an install to update your resolutions`); } + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: unknown, opts: ResolveOptions): Promise { + throw new ReportError(MessageName.MISSING_LOCKFILE_ENTRY, `This package doesn't seem to be present in your lockfile; try to make an install to update your resolutions`); + } + async resolve(locator: Locator, opts: ResolveOptions): Promise { throw new ReportError(MessageName.MISSING_LOCKFILE_ENTRY, `This package doesn't seem to be present in your lockfile; try to make an install to update your resolutions`); } diff --git a/packages/yarnpkg-core/sources/VirtualResolver.ts b/packages/yarnpkg-core/sources/VirtualResolver.ts index e322cc6cb5b6..48e9584f9a5f 100644 --- a/packages/yarnpkg-core/sources/VirtualResolver.ts +++ b/packages/yarnpkg-core/sources/VirtualResolver.ts @@ -1,5 +1,5 @@ import {Resolver, ResolveOptions, MinimalResolveOptions} from './Resolver'; -import {Descriptor, Locator} from './types'; +import {Descriptor, Locator, DescriptorHash, Package} from './types'; export class VirtualResolver implements Resolver { static protocol = `virtual:`; @@ -54,6 +54,14 @@ export class VirtualResolver implements Resolver { throw new Error(`Assertion failed: calling "getCandidates" on a virtual descriptor is unsupported`); } + async getSatisfying(descriptor: Descriptor, candidates: Array, dependencies: Map, opts: ResolveOptions): Promise { + // It's unsupported because packages inside the dependency tree should + // only become virtual AFTER they have all been resolved, by which point + // you shouldn't need to call `getSatisfying` anymore. + + throw new Error(`Assertion failed: calling "getSatisfying" on a virtual descriptor is unsupported`); + } + async resolve(locator: Locator, opts: ResolveOptions): Promise { // It's unsupported because packages inside the dependency tree should // only become virtual AFTER they have all been resolved, by which point diff --git a/packages/yarnpkg-core/sources/WorkspaceResolver.ts b/packages/yarnpkg-core/sources/WorkspaceResolver.ts index 284472283851..6e05370f629b 100644 --- a/packages/yarnpkg-core/sources/WorkspaceResolver.ts +++ b/packages/yarnpkg-core/sources/WorkspaceResolver.ts @@ -1,8 +1,9 @@ import {PortablePath} from '@yarnpkg/fslib'; import {Resolver, ResolveOptions, MinimalResolveOptions} from './Resolver'; -import {Descriptor, Locator} from './types'; +import {Descriptor, Locator, DescriptorHash, Package} from './types'; import {LinkType} from './types'; +import {structUtils} from '.'; export class WorkspaceResolver implements Resolver { static protocol = `workspace:`; @@ -43,6 +44,14 @@ export class WorkspaceResolver implements Resolver { return [workspace.anchoredLocator]; } + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + const workspace = opts.project.getWorkspaceByDescriptor(descriptor); + + return references + .filter(reference => reference === workspace.anchoredLocator.reference) + .map(reference => structUtils.makeLocator(descriptor, reference)); + } + async resolve(locator: Locator, opts: ResolveOptions) { const workspace = opts.project.getWorkspaceByCwd(locator.reference.slice(WorkspaceResolver.protocol.length) as PortablePath); From cc916622231ebfc641b538dc733b57116f0e9d47 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Fri, 21 Aug 2020 01:46:33 +0300 Subject: [PATCH 12/24] refactor: fix cyclic dependency --- packages/plugin-essentials/sources/commands/dedupe.ts | 2 +- packages/yarnpkg-core/sources/WorkspaceResolver.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index 557eff8defe2..b452eb7b22e9 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -4,7 +4,7 @@ * - https://github.com/eps1lon/yarn-plugin-deduplicate * * Goals of the `dedupe` command: - * - the deduplication algorithms shouldn't depend on semver; they should instead use the resolver candidate system + * - the deduplication algorithms shouldn't depend on semver; they should instead use the resolver `getSatisfying` system * - the deduplication should happen concurrently */ diff --git a/packages/yarnpkg-core/sources/WorkspaceResolver.ts b/packages/yarnpkg-core/sources/WorkspaceResolver.ts index 6e05370f629b..fbb95e9837fa 100644 --- a/packages/yarnpkg-core/sources/WorkspaceResolver.ts +++ b/packages/yarnpkg-core/sources/WorkspaceResolver.ts @@ -1,9 +1,9 @@ import {PortablePath} from '@yarnpkg/fslib'; import {Resolver, ResolveOptions, MinimalResolveOptions} from './Resolver'; +import * as structUtils from './structUtils'; import {Descriptor, Locator, DescriptorHash, Package} from './types'; import {LinkType} from './types'; -import {structUtils} from '.'; export class WorkspaceResolver implements Resolver { static protocol = `workspace:`; From 51b342eefc7768b74b07e7b75460fb3e125c0bc1 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sat, 22 Aug 2020 23:17:29 +0300 Subject: [PATCH 13/24] Update packages/plugin-essentials/sources/commands/dedupe.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Maël Nison --- packages/plugin-essentials/sources/commands/dedupe.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index b452eb7b22e9..b6795fe85059 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -161,7 +161,7 @@ export default class DedupeCommand extends BaseCommand { - \`highest\`: Reuses (where possible) the locators with the highest versions. This means that dependencies can only be upgraded, never downgraded. It's also guaranteed that it never takes more than a single pass to dedupe the entire dependency tree. - **Note:** Even though it never produces a wrong dependency tree, this command should be used with caution, as it modifies the dependency tree, which can sometimes cause problems when packages specify wrong dependency ranges. It is recommended to also review the changes manually. + **Note:** Even though it never produces a wrong dependency tree, this command should be used with caution, as it modifies the dependency tree, which can sometimes cause problems when packages strictly follow semver recommandations. It is recommended to also review the changes manually. If set, the \`-c,--check\` flag will only report the found duplicates, without persisting the modified dependency tree. From 4379dbe0a4aa69879692cfa0f1e6e4a1fd625bfc Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sat, 22 Aug 2020 23:17:43 +0300 Subject: [PATCH 14/24] Update packages/plugin-essentials/sources/commands/dedupe.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Maël Nison --- packages/plugin-essentials/sources/commands/dedupe.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index b6795fe85059..bb49017f9db1 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -163,7 +163,7 @@ export default class DedupeCommand extends BaseCommand { **Note:** Even though it never produces a wrong dependency tree, this command should be used with caution, as it modifies the dependency tree, which can sometimes cause problems when packages strictly follow semver recommandations. It is recommended to also review the changes manually. - If set, the \`-c,--check\` flag will only report the found duplicates, without persisting the modified dependency tree. + If set, the \`-c,--check\` flag will only report the found duplicates, without persisting the modified dependency tree. If changes are found, the command will exit with a non-zero exit code, making it suitable for CI purposes. This command accepts glob patterns as arguments (if valid Idents and supported by [micromatch](https://github.com/micromatch/micromatch)). Make sure to escape the patterns, to prevent your own shell from trying to expand them. From d637d8ece10637a10e5bf1db9e261a9f562a6531 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sat, 22 Aug 2020 23:33:32 +0300 Subject: [PATCH 15/24] refactor: dedupeSkip -> null --- .../sources/commands/dedupe.ts | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index bb49017f9db1..77a0fed31676 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -15,13 +15,11 @@ import {Command} import micromatch from 'micromatch'; import * as yup from 'yup'; -export const dedupeSkip = Symbol(`dedupeSkip`); - export type DedupePromise = Promise<{ descriptor: Descriptor, currentPackage: Package, updatedPackage: Package, -} | typeof dedupeSkip>; +} | null>; export type DedupeAlgorithm = (project: Project, patterns: Array, opts: { resolver: Resolver, @@ -56,32 +54,32 @@ export const DEDUPE_ALGORITHMS: Record = { return Array.from(project.storedDescriptors.values(), async descriptor => { if (structUtils.isVirtualDescriptor(descriptor)) - return dedupeSkip; + return null; if (patterns.length && !micromatch.isMatch(structUtils.stringifyIdent(descriptor), patterns)) - return dedupeSkip; + return null; const currentResolution = project.storedResolutions.get(descriptor.descriptorHash); if (typeof currentResolution === `undefined`) - return dedupeSkip; + return null; // We only care about resolutions that are stored in the lockfile const currentPackage = project.originalPackages.get(currentResolution); if (typeof currentPackage === `undefined`) - return dedupeSkip; + return null; // No need to try deduping packages that are not persisted, // they will be resolved again anyways if (!resolver.shouldPersistResolution(currentPackage, resolveOptions)) - return dedupeSkip; + return null; const locators = locatorsByIdent.get(descriptor.identHash); if (typeof locators === `undefined`) - return dedupeSkip; + return null; // No need to choose when there's only one possibility if (locators.size === 1) - return dedupeSkip; + return null; const references = [...locators].map(locatorHash => { const pkg = project.originalPackages.get(locatorHash); @@ -110,17 +108,17 @@ export const DEDUPE_ALGORITHMS: Record = { const bestCandidate = candidates[0]; if (typeof bestCandidate === `undefined`) - return dedupeSkip; + return null; const updatedResolution = bestCandidate.locatorHash; // We only care about resolutions that are stored in the lockfile const updatedPackage = project.originalPackages.get(updatedResolution); if (typeof updatedPackage === `undefined`) - return dedupeSkip; + return null; if (updatedResolution === currentResolution) - return dedupeSkip; + return null; return {descriptor, currentPackage, updatedPackage}; }); @@ -273,7 +271,7 @@ export async function dedupe({strategy, project, patterns, cache, report}: Dedup dedupePromises.map(dedupePromise => dedupePromise .then(dedupe => { - if (dedupe === dedupeSkip) + if (dedupe === null) return; dedupedPackageCount++; From 1a705f57bc142ac7ca64389046664d0955570a9c Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sun, 23 Aug 2020 00:01:50 +0300 Subject: [PATCH 16/24] refactor: skip -> assertion --- packages/plugin-essentials/sources/commands/dedupe.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index 77a0fed31676..9e6c92a2db88 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -61,7 +61,7 @@ export const DEDUPE_ALGORITHMS: Record = { const currentResolution = project.storedResolutions.get(descriptor.descriptorHash); if (typeof currentResolution === `undefined`) - return null; + throw new Error(`Assertion failed: The resolution (${descriptor.descriptorHash}) should have been registered`); // We only care about resolutions that are stored in the lockfile const currentPackage = project.originalPackages.get(currentResolution); From cab447ed7552ec02c42ef656a7afab8c788d6a32 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sun, 23 Aug 2020 00:15:30 +0300 Subject: [PATCH 17/24] refactor: add note about virtual packages --- packages/plugin-essentials/sources/commands/dedupe.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index 9e6c92a2db88..78ba04c6576b 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -6,6 +6,10 @@ * Goals of the `dedupe` command: * - the deduplication algorithms shouldn't depend on semver; they should instead use the resolver `getSatisfying` system * - the deduplication should happen concurrently + * + * Note: We don't restore the install state because we already have everything we need inside the + * lockfile. Because of this, we use `project.originalPackages` instead of `project.storedPackages` + * (which also provides a safe-guard in case virtual descriptors ever make their way into the dedupe algorithm). */ import {BaseCommand} from '@yarnpkg/cli'; @@ -53,9 +57,6 @@ export const DEDUPE_ALGORITHMS: Record = { } return Array.from(project.storedDescriptors.values(), async descriptor => { - if (structUtils.isVirtualDescriptor(descriptor)) - return null; - if (patterns.length && !micromatch.isMatch(structUtils.stringifyIdent(descriptor), patterns)) return null; @@ -64,6 +65,7 @@ export const DEDUPE_ALGORITHMS: Record = { throw new Error(`Assertion failed: The resolution (${descriptor.descriptorHash}) should have been registered`); // We only care about resolutions that are stored in the lockfile + // (we shouldn't accidentally try deduping virtual packages) const currentPackage = project.originalPackages.get(currentResolution); if (typeof currentPackage === `undefined`) return null; From d6b9a58384c2f5cf7f077f54d0763d7e494fb9c2 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sun, 23 Aug 2020 00:22:27 +0300 Subject: [PATCH 18/24] refactor: skip ->assertion failed --- packages/plugin-essentials/sources/commands/dedupe.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index 78ba04c6576b..f7e80ec4504f 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -114,10 +114,9 @@ export const DEDUPE_ALGORITHMS: Record = { const updatedResolution = bestCandidate.locatorHash; - // We only care about resolutions that are stored in the lockfile const updatedPackage = project.originalPackages.get(updatedResolution); if (typeof updatedPackage === `undefined`) - return null; + throw new Error(`Assertion failed: The package (${updatedResolution}) should have been registered`); if (updatedResolution === currentResolution) return null; From 9eb67d8ca6c189a26eef6c53b2173a32d24f4240 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sun, 23 Aug 2020 13:25:44 +0300 Subject: [PATCH 19/24] refactor: disable getSatisfying for some resolvers --- packages/plugin-exec/sources/ExecResolver.ts | 48 ++++++--------- packages/plugin-file/sources/FileResolver.ts | 60 ++++++++----------- .../sources/TarballFileResolver.ts | 22 ++----- packages/plugin-git/sources/GitResolver.ts | 16 +---- .../sources/TarballHttpResolver.ts | 6 +- packages/plugin-link/sources/LinkResolver.ts | 16 +---- .../plugin-link/sources/RawLinkResolver.ts | 16 +---- packages/plugin-npm/sources/NpmTagResolver.ts | 2 +- .../plugin-patch/sources/PatchResolver.ts | 34 ++++------- .../sources/LegacyMigrationResolver.ts | 11 +--- .../yarnpkg-core/sources/LockfileResolver.ts | 14 +---- packages/yarnpkg-core/sources/Resolver.ts | 5 +- .../yarnpkg-core/sources/WorkspaceResolver.ts | 6 +- 13 files changed, 80 insertions(+), 176 deletions(-) diff --git a/packages/plugin-exec/sources/ExecResolver.ts b/packages/plugin-exec/sources/ExecResolver.ts index ff9d96394251..d1b61705b47d 100644 --- a/packages/plugin-exec/sources/ExecResolver.ts +++ b/packages/plugin-exec/sources/ExecResolver.ts @@ -39,17 +39,29 @@ export class ExecResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: Map, opts: ResolveOptions) { - const locator = await this.getCandidateForDescriptor(descriptor, opts); + if (!opts.fetchOptions) + throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); + + const {path, parentLocator} = execUtils.parseSpec(descriptor.range); + + if (parentLocator === null) + throw new Error(`Assertion failed: The descriptor should have been bound`); - return [locator]; + const generatorFile = await execUtils.loadGeneratorFile(structUtils.makeRange({ + protocol: PROTOCOL, + source: path, + selector: path, + params: { + locator: structUtils.stringifyLocator(parentLocator), + }, + }), PROTOCOL, opts.fetchOptions); + const generatorHash = hashUtils.makeHash(`${CACHE_VERSION}`, generatorFile).slice(0, 6); + + return [execUtils.makeLocator(descriptor, {parentLocator, path, generatorHash, protocol: PROTOCOL})]; } async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - const {reference: execReference} = await this.getCandidateForDescriptor(descriptor, opts); - - return references - .filter(reference => reference === execReference) - .map(reference => structUtils.makeLocator(descriptor, reference)); + return null; } async resolve(locator: Locator, opts: ResolveOptions) { @@ -79,26 +91,4 @@ export class ExecResolver implements Resolver { bin: manifest.bin, }; } - - private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { - if (!opts.fetchOptions) - throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); - - const {path, parentLocator} = execUtils.parseSpec(descriptor.range); - - if (parentLocator === null) - throw new Error(`Assertion failed: The descriptor should have been bound`); - - const generatorFile = await execUtils.loadGeneratorFile(structUtils.makeRange({ - protocol: PROTOCOL, - source: path, - selector: path, - params: { - locator: structUtils.stringifyLocator(parentLocator), - }, - }), PROTOCOL, opts.fetchOptions); - const generatorHash = hashUtils.makeHash(`${CACHE_VERSION}`, generatorFile).slice(0, 6); - - return execUtils.makeLocator(descriptor, {parentLocator, path, generatorHash, protocol: PROTOCOL}); - } } diff --git a/packages/plugin-file/sources/FileResolver.ts b/packages/plugin-file/sources/FileResolver.ts index 875f4e1b4066..28d0e161419a 100644 --- a/packages/plugin-file/sources/FileResolver.ts +++ b/packages/plugin-file/sources/FileResolver.ts @@ -45,17 +45,35 @@ export class FileResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) { - const locator = await this.getCandidateForDescriptor(descriptor, opts); + if (!opts.fetchOptions) + throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); + + const {path, parentLocator} = fileUtils.parseSpec(descriptor.range); + + if (parentLocator === null) + throw new Error(`Assertion failed: The descriptor should have been bound`); - return [locator]; + const archiveBuffer = await fileUtils.makeBufferFromLocator( + structUtils.makeLocator(descriptor, + structUtils.makeRange({ + protocol: PROTOCOL, + source: path, + selector: path, + params: { + locator: structUtils.stringifyLocator(parentLocator), + }, + }) + ), + {protocol: PROTOCOL, fetchOptions: opts.fetchOptions} + ); + + const folderHash = hashUtils.makeHash(`${CACHE_VERSION}`, archiveBuffer).slice(0, 6); + + return [fileUtils.makeLocator(descriptor, {parentLocator, path, folderHash, protocol: PROTOCOL})]; } async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - const {reference: fileReference} = await this.getCandidateForDescriptor(descriptor, opts); - - return references - .filter(reference => reference === fileReference) - .map(reference => structUtils.makeLocator(descriptor, reference)); + return null; } async resolve(locator: Locator, opts: ResolveOptions) { @@ -85,32 +103,4 @@ export class FileResolver implements Resolver { bin: manifest.bin, }; } - - private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { - if (!opts.fetchOptions) - throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); - - const {path, parentLocator} = fileUtils.parseSpec(descriptor.range); - - if (parentLocator === null) - throw new Error(`Assertion failed: The descriptor should have been bound`); - - const archiveBuffer = await fileUtils.makeBufferFromLocator( - structUtils.makeLocator(descriptor, - structUtils.makeRange({ - protocol: PROTOCOL, - source: path, - selector: path, - params: { - locator: structUtils.stringifyLocator(parentLocator), - }, - }) - ), - {protocol: PROTOCOL, fetchOptions: opts.fetchOptions} - ); - - const folderHash = hashUtils.makeHash(`${CACHE_VERSION}`, archiveBuffer).slice(0, 6); - - return fileUtils.makeLocator(descriptor, {parentLocator, path, folderHash, protocol: PROTOCOL}); - } } diff --git a/packages/plugin-file/sources/TarballFileResolver.ts b/packages/plugin-file/sources/TarballFileResolver.ts index 48956dcbb1dd..d75c53b02176 100644 --- a/packages/plugin-file/sources/TarballFileResolver.ts +++ b/packages/plugin-file/sources/TarballFileResolver.ts @@ -48,17 +48,16 @@ export class TarballFileResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) { - const locator = await this.getCandidateForDescriptor(descriptor, opts); + let path = descriptor.range; + + if (path.startsWith(PROTOCOL)) + path = path.slice(PROTOCOL.length); - return [locator]; + return [structUtils.makeLocator(descriptor, `${PROTOCOL}${npath.toPortablePath(path)}`)]; } async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - const {reference: tarballFileReference} = await this.getCandidateForDescriptor(descriptor, opts); - - return references - .filter(reference => reference === tarballFileReference) - .map(reference => structUtils.makeLocator(descriptor, reference)); + return null; } async resolve(locator: Locator, opts: ResolveOptions) { @@ -88,13 +87,4 @@ export class TarballFileResolver implements Resolver { bin: manifest.bin, }; } - - private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { - let path = descriptor.range; - - if (path.startsWith(PROTOCOL)) - path = path.slice(PROTOCOL.length); - - return structUtils.makeLocator(descriptor, `${PROTOCOL}${npath.toPortablePath(path)}`); - } } diff --git a/packages/plugin-git/sources/GitResolver.ts b/packages/plugin-git/sources/GitResolver.ts index 9dda0ca7c504..712a5055b6f5 100644 --- a/packages/plugin-git/sources/GitResolver.ts +++ b/packages/plugin-git/sources/GitResolver.ts @@ -27,17 +27,14 @@ export class GitResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) { - const locator = await this.getCandidateForDescriptor(descriptor, opts); + const reference = await gitUtils.resolveUrl(descriptor.range, opts.project.configuration); + const locator = structUtils.makeLocator(descriptor, reference); return [locator]; } async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - const {reference: gitReference} = await this.getCandidateForDescriptor(descriptor, opts); - - return references - .filter(reference => reference === gitReference) - .map(reference => structUtils.makeLocator(descriptor, reference)); + return null; } async resolve(locator: Locator, opts: ResolveOptions) { @@ -67,11 +64,4 @@ export class GitResolver implements Resolver { bin: manifest.bin, }; } - - private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { - const reference = await gitUtils.resolveUrl(descriptor.range, opts.project.configuration); - const locator = structUtils.makeLocator(descriptor, reference); - - return locator; - } } diff --git a/packages/plugin-http/sources/TarballHttpResolver.ts b/packages/plugin-http/sources/TarballHttpResolver.ts index 0d6672572099..1fd4b5816256 100644 --- a/packages/plugin-http/sources/TarballHttpResolver.ts +++ b/packages/plugin-http/sources/TarballHttpResolver.ts @@ -43,11 +43,7 @@ export class TarballHttpResolver implements Resolver { } async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - const tarballReference = structUtils.convertDescriptorToLocator(descriptor).reference; - - return references - .filter(reference => reference === tarballReference) - .map(reference => structUtils.makeLocator(descriptor, reference)); + return null; } async resolve(locator: Locator, opts: ResolveOptions) { diff --git a/packages/plugin-link/sources/LinkResolver.ts b/packages/plugin-link/sources/LinkResolver.ts index 96a219b0cf8b..bfc3104659eb 100644 --- a/packages/plugin-link/sources/LinkResolver.ts +++ b/packages/plugin-link/sources/LinkResolver.ts @@ -36,17 +36,13 @@ export class LinkResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) { - const locator = await this.getCandidateForDescriptor(descriptor, opts); + const path = descriptor.range.slice(LINK_PROTOCOL.length); - return [locator]; + return [structUtils.makeLocator(descriptor, `${LINK_PROTOCOL}${npath.toPortablePath(path)}`)]; } async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - const {reference: linkReference} = await this.getCandidateForDescriptor(descriptor, opts); - - return references - .filter(reference => reference === linkReference) - .map(reference => structUtils.makeLocator(descriptor, reference)); + return null; } async resolve(locator: Locator, opts: ResolveOptions): Promise { @@ -76,10 +72,4 @@ export class LinkResolver implements Resolver { bin: manifest.bin, }; } - - private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { - const path = descriptor.range.slice(LINK_PROTOCOL.length); - - return structUtils.makeLocator(descriptor, `${LINK_PROTOCOL}${npath.toPortablePath(path)}`); - } } diff --git a/packages/plugin-link/sources/RawLinkResolver.ts b/packages/plugin-link/sources/RawLinkResolver.ts index 59cfbe7f260c..4e847f0eb71a 100644 --- a/packages/plugin-link/sources/RawLinkResolver.ts +++ b/packages/plugin-link/sources/RawLinkResolver.ts @@ -36,17 +36,13 @@ export class RawLinkResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) { - const locator = await this.getCandidateForDescriptor(descriptor, opts); + const path = descriptor.range.slice(RAW_LINK_PROTOCOL.length); - return [locator]; + return [structUtils.makeLocator(descriptor, `${RAW_LINK_PROTOCOL}${npath.toPortablePath(path)}`)]; } async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - const {reference: rawLinkReference} = await this.getCandidateForDescriptor(descriptor, opts); - - return references - .filter(reference => reference === rawLinkReference) - .map(reference => structUtils.makeLocator(descriptor, reference)); + return null; } async resolve(locator: Locator, opts: ResolveOptions) { @@ -67,10 +63,4 @@ export class RawLinkResolver implements Resolver { bin: new Map(), }; } - - private async getCandidateForDescriptor(descriptor: Descriptor, opts: ResolveOptions) { - const path = descriptor.range.slice(RAW_LINK_PROTOCOL.length); - - return structUtils.makeLocator(descriptor, `${RAW_LINK_PROTOCOL}${npath.toPortablePath(path)}`); - } } diff --git a/packages/plugin-npm/sources/NpmTagResolver.ts b/packages/plugin-npm/sources/NpmTagResolver.ts index bef62dfa682b..171d2e0c9ec3 100644 --- a/packages/plugin-npm/sources/NpmTagResolver.ts +++ b/packages/plugin-npm/sources/NpmTagResolver.ts @@ -66,7 +66,7 @@ export class NpmTagResolver implements Resolver { async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { // We can't statically know if a tag resolves to a specific version without using the network - return []; + return null; } async resolve(locator: Locator, opts: ResolveOptions): Promise { diff --git a/packages/plugin-patch/sources/PatchResolver.ts b/packages/plugin-patch/sources/PatchResolver.ts index 79a702d1a1d0..326a39f995d1 100644 --- a/packages/plugin-patch/sources/PatchResolver.ts +++ b/packages/plugin-patch/sources/PatchResolver.ts @@ -46,27 +46,6 @@ export class PatchResolver implements Resolver { } async getCandidates(descriptor: Descriptor, dependencies: Map, opts: ResolveOptions) { - const locator = await this.getCandidateForDescriptor(descriptor, dependencies, opts); - - return [locator]; - } - - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - const {reference: patchReference} = await this.getCandidateForDescriptor(descriptor, dependencies, opts); - - return references - .filter(reference => reference === patchReference) - .map(reference => structUtils.makeLocator(descriptor, reference)); - } - - async resolve(locator: Locator, opts: ResolveOptions): Promise { - const {sourceLocator} = patchUtils.parseLocator(locator); - const sourcePkg = await opts.resolver.resolve(sourceLocator, opts); - - return {...sourcePkg, ...locator}; - } - - private async getCandidateForDescriptor(descriptor: Descriptor, dependencies: Map, opts: ResolveOptions) { if (!opts.fetchOptions) throw new Error(`Assertion failed: This resolver cannot be used unless a fetcher is configured`); @@ -79,6 +58,17 @@ export class PatchResolver implements Resolver { const patchHash = hashUtils.makeHash(`${CACHE_VERSION}`, ...patchFiles).slice(0, 6); - return patchUtils.makeLocator(descriptor, {parentLocator, sourcePackage, patchPaths, patchHash}); + return [patchUtils.makeLocator(descriptor, {parentLocator, sourcePackage, patchPaths, patchHash})]; + } + + async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + return null; + } + + async resolve(locator: Locator, opts: ResolveOptions): Promise { + const {sourceLocator} = patchUtils.parseLocator(locator); + const sourcePkg = await opts.resolver.resolve(sourceLocator, opts); + + return {...sourcePkg, ...locator}; } } diff --git a/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts b/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts index 1070554bd8ec..cc2bfdf8aae6 100644 --- a/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts +++ b/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts @@ -116,16 +116,7 @@ export class LegacyMigrationResolver implements Resolver { } async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - if (!this.resolutions) - throw new Error(`Assertion failed: The resolution store should have been setup`); - - const resolution = this.resolutions.get(descriptor.descriptorHash); - if (!resolution) - throw new Error(`Assertion failed: The resolution should have been registered`); - - return references - .filter(reference => reference === resolution.reference) - .map(reference => structUtils.makeLocator(descriptor, reference)); + return null; } async resolve(locator: Locator, opts: ResolveOptions): Promise { diff --git a/packages/yarnpkg-core/sources/LockfileResolver.ts b/packages/yarnpkg-core/sources/LockfileResolver.ts index e0ed90f8fe80..fa8755c276dd 100644 --- a/packages/yarnpkg-core/sources/LockfileResolver.ts +++ b/packages/yarnpkg-core/sources/LockfileResolver.ts @@ -52,19 +52,7 @@ export class LockfileResolver implements Resolver { } async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - const convertedPkg = opts.project.originalPackages.get(structUtils.convertDescriptorToLocator(descriptor).locatorHash); - - const resolution = opts.project.storedResolutions.get(descriptor.descriptorHash); - const resolutionPkg = typeof resolution !== `undefined` - ? opts.project.originalPackages.get(resolution) - : undefined; - - return references - .filter(reference => [ - convertedPkg?.reference, - resolutionPkg?.reference, - ].includes(reference)) - .map(reference => structUtils.makeLocator(descriptor, reference)); + return null; } async resolve(locator: Locator, opts: ResolveOptions) { diff --git a/packages/yarnpkg-core/sources/Resolver.ts b/packages/yarnpkg-core/sources/Resolver.ts index 33ff68004582..0cf91ca29bba 100644 --- a/packages/yarnpkg-core/sources/Resolver.ts +++ b/packages/yarnpkg-core/sources/Resolver.ts @@ -138,12 +138,15 @@ export interface Resolver { * locators are first. This will cause the resolution algorithm to prioritize * them if possible (it doesn't guarantee that they'll end up being used). * + * If the operation is unsupported by the resolver (i.e. if it can't be statically + * determined which references satisfy the target descriptor), `null` should be returned. + * * @param descriptor The target descriptor. * @param references The candidate references. * @param dependencies The resolution dependencies and their resolutions. * @param opts The resolution options. */ - getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions): Promise>; + getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions): Promise | null>; /** * This function will, given a locator, return the full package definition diff --git a/packages/yarnpkg-core/sources/WorkspaceResolver.ts b/packages/yarnpkg-core/sources/WorkspaceResolver.ts index fbb95e9837fa..edb0231218f9 100644 --- a/packages/yarnpkg-core/sources/WorkspaceResolver.ts +++ b/packages/yarnpkg-core/sources/WorkspaceResolver.ts @@ -45,11 +45,7 @@ export class WorkspaceResolver implements Resolver { } async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - const workspace = opts.project.getWorkspaceByDescriptor(descriptor); - - return references - .filter(reference => reference === workspace.anchoredLocator.reference) - .map(reference => structUtils.makeLocator(descriptor, reference)); + return null; } async resolve(locator: Locator, opts: ResolveOptions) { From aaf702fa44b9bfb2a730e6760876f18db31b3435 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sun, 23 Aug 2020 13:36:25 +0300 Subject: [PATCH 20/24] refactor: remove resolutionDependencies --- .../sources/commands/dedupe.ts | 19 ++----------------- packages/plugin-exec/sources/ExecResolver.ts | 2 +- packages/plugin-file/sources/FileResolver.ts | 14 +++++++------- .../sources/TarballFileResolver.ts | 14 +++++++------- packages/plugin-git/sources/GitResolver.ts | 12 ++++++------ .../sources/TarballHttpResolver.ts | 12 ++++++------ packages/plugin-link/sources/LinkResolver.ts | 14 +++++++------- .../plugin-link/sources/RawLinkResolver.ts | 14 +++++++------- .../plugin-npm/sources/NpmRemapResolver.ts | 4 ++-- .../plugin-npm/sources/NpmSemverResolver.ts | 2 +- packages/plugin-npm/sources/NpmTagResolver.ts | 2 +- .../plugin-patch/sources/PatchResolver.ts | 2 +- .../sources/LegacyMigrationResolver.ts | 4 ++-- .../yarnpkg-core/sources/LockfileResolver.ts | 4 ++-- .../yarnpkg-core/sources/MultiResolver.ts | 4 ++-- .../yarnpkg-core/sources/ProtocolResolver.ts | 4 ++-- packages/yarnpkg-core/sources/Resolver.ts | 3 +-- .../sources/RunInstallPleaseResolver.ts | 2 +- .../yarnpkg-core/sources/VirtualResolver.ts | 4 ++-- .../yarnpkg-core/sources/WorkspaceResolver.ts | 5 ++--- 20 files changed, 62 insertions(+), 79 deletions(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index f7e80ec4504f..32f3c481408e 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -91,24 +91,9 @@ export const DEDUPE_ALGORITHMS: Record = { return pkg.reference; }); - const resolutionDependencies = resolver.getResolutionDependencies(descriptor, resolveOptions); - const dependencies = new Map( - resolutionDependencies.map(({descriptorHash}) => { - const resolution = project.storedResolutions.get(descriptorHash); - if (typeof resolution === `undefined`) - throw new Error(`Assertion failed: The resolution (${descriptorHash}) should have been registered`); + const candidates = await resolver.getSatisfying(descriptor, references, resolveOptions); - const pkg = project.originalPackages.get(resolution); - if (typeof pkg === `undefined`) - throw new Error(`Assertion failed: The package (${resolution}) should have been registered`); - - return [descriptorHash, pkg] as const; - }) - ); - - const candidates = await resolver.getSatisfying(descriptor, references, dependencies, resolveOptions); - - const bestCandidate = candidates[0]; + const bestCandidate = candidates?.[0]; if (typeof bestCandidate === `undefined`) return null; diff --git a/packages/plugin-exec/sources/ExecResolver.ts b/packages/plugin-exec/sources/ExecResolver.ts index d1b61705b47d..7e10f7eee785 100644 --- a/packages/plugin-exec/sources/ExecResolver.ts +++ b/packages/plugin-exec/sources/ExecResolver.ts @@ -60,7 +60,7 @@ export class ExecResolver implements Resolver { return [execUtils.makeLocator(descriptor, {parentLocator, path, generatorHash, protocol: PROTOCOL})]; } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { return null; } diff --git a/packages/plugin-file/sources/FileResolver.ts b/packages/plugin-file/sources/FileResolver.ts index 28d0e161419a..5b5bc0e6297e 100644 --- a/packages/plugin-file/sources/FileResolver.ts +++ b/packages/plugin-file/sources/FileResolver.ts @@ -1,10 +1,10 @@ -import {miscUtils, structUtils, hashUtils, DescriptorHash, Package} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; -import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; +import {miscUtils, structUtils, hashUtils} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; +import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; -import {FILE_REGEXP, PROTOCOL} from './constants'; -import * as fileUtils from './fileUtils'; +import {FILE_REGEXP, PROTOCOL} from './constants'; +import * as fileUtils from './fileUtils'; // We use this for the folders to be regenerated without bumping the whole cache const CACHE_VERSION = 1; @@ -72,7 +72,7 @@ export class FileResolver implements Resolver { return [fileUtils.makeLocator(descriptor, {parentLocator, path, folderHash, protocol: PROTOCOL})]; } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { return null; } diff --git a/packages/plugin-file/sources/TarballFileResolver.ts b/packages/plugin-file/sources/TarballFileResolver.ts index d75c53b02176..bbc012a4167a 100644 --- a/packages/plugin-file/sources/TarballFileResolver.ts +++ b/packages/plugin-file/sources/TarballFileResolver.ts @@ -1,10 +1,10 @@ -import {Resolver, ResolveOptions, MinimalResolveOptions, DescriptorHash, Package} from '@yarnpkg/core'; -import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {miscUtils, structUtils} from '@yarnpkg/core'; -import {npath} from '@yarnpkg/fslib'; +import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; +import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {miscUtils, structUtils} from '@yarnpkg/core'; +import {npath} from '@yarnpkg/fslib'; -import {FILE_REGEXP, TARBALL_REGEXP, PROTOCOL} from './constants'; +import {FILE_REGEXP, TARBALL_REGEXP, PROTOCOL} from './constants'; export class TarballFileResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -56,7 +56,7 @@ export class TarballFileResolver implements Resolver { return [structUtils.makeLocator(descriptor, `${PROTOCOL}${npath.toPortablePath(path)}`)]; } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { return null; } diff --git a/packages/plugin-git/sources/GitResolver.ts b/packages/plugin-git/sources/GitResolver.ts index 712a5055b6f5..11e306895761 100644 --- a/packages/plugin-git/sources/GitResolver.ts +++ b/packages/plugin-git/sources/GitResolver.ts @@ -1,9 +1,9 @@ -import {Resolver, ResolveOptions, MinimalResolveOptions, DescriptorHash, Package} from '@yarnpkg/core'; -import {miscUtils, structUtils} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; +import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; +import {miscUtils, structUtils} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; -import * as gitUtils from './gitUtils'; +import * as gitUtils from './gitUtils'; export class GitResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -33,7 +33,7 @@ export class GitResolver implements Resolver { return [locator]; } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { return null; } diff --git a/packages/plugin-http/sources/TarballHttpResolver.ts b/packages/plugin-http/sources/TarballHttpResolver.ts index 1fd4b5816256..cff3a952a86e 100644 --- a/packages/plugin-http/sources/TarballHttpResolver.ts +++ b/packages/plugin-http/sources/TarballHttpResolver.ts @@ -1,9 +1,9 @@ -import {Resolver, ResolveOptions, MinimalResolveOptions, DescriptorHash, Package} from '@yarnpkg/core'; -import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {miscUtils, structUtils} from '@yarnpkg/core'; +import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; +import {Descriptor, Locator, Manifest} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {miscUtils, structUtils} from '@yarnpkg/core'; -import {PROTOCOL_REGEXP, TARBALL_REGEXP} from './constants'; +import {PROTOCOL_REGEXP, TARBALL_REGEXP} from './constants'; export class TarballHttpResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -42,7 +42,7 @@ export class TarballHttpResolver implements Resolver { return [structUtils.convertDescriptorToLocator(descriptor)]; } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { return null; } diff --git a/packages/plugin-link/sources/LinkResolver.ts b/packages/plugin-link/sources/LinkResolver.ts index bfc3104659eb..d5ef26df278d 100644 --- a/packages/plugin-link/sources/LinkResolver.ts +++ b/packages/plugin-link/sources/LinkResolver.ts @@ -1,10 +1,10 @@ -import {Resolver, ResolveOptions, MinimalResolveOptions, DescriptorHash} from '@yarnpkg/core'; -import {Descriptor, Locator, Manifest, Package} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {miscUtils, structUtils} from '@yarnpkg/core'; -import {npath} from '@yarnpkg/fslib'; +import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; +import {Descriptor, Locator, Manifest, Package} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {miscUtils, structUtils} from '@yarnpkg/core'; +import {npath} from '@yarnpkg/fslib'; -import {LINK_PROTOCOL} from './constants'; +import {LINK_PROTOCOL} from './constants'; export class LinkResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -41,7 +41,7 @@ export class LinkResolver implements Resolver { return [structUtils.makeLocator(descriptor, `${LINK_PROTOCOL}${npath.toPortablePath(path)}`)]; } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { return null; } diff --git a/packages/plugin-link/sources/RawLinkResolver.ts b/packages/plugin-link/sources/RawLinkResolver.ts index 4e847f0eb71a..20b89e0fa40b 100644 --- a/packages/plugin-link/sources/RawLinkResolver.ts +++ b/packages/plugin-link/sources/RawLinkResolver.ts @@ -1,10 +1,10 @@ -import {Resolver, ResolveOptions, MinimalResolveOptions, DescriptorHash, Package} from '@yarnpkg/core'; -import {Descriptor, Locator} from '@yarnpkg/core'; -import {LinkType} from '@yarnpkg/core'; -import {structUtils} from '@yarnpkg/core'; -import {npath} from '@yarnpkg/fslib'; +import {Resolver, ResolveOptions, MinimalResolveOptions} from '@yarnpkg/core'; +import {Descriptor, Locator} from '@yarnpkg/core'; +import {LinkType} from '@yarnpkg/core'; +import {structUtils} from '@yarnpkg/core'; +import {npath} from '@yarnpkg/fslib'; -import {RAW_LINK_PROTOCOL} from './constants'; +import {RAW_LINK_PROTOCOL} from './constants'; export class RawLinkResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -41,7 +41,7 @@ export class RawLinkResolver implements Resolver { return [structUtils.makeLocator(descriptor, `${RAW_LINK_PROTOCOL}${npath.toPortablePath(path)}`)]; } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { return null; } diff --git a/packages/plugin-npm/sources/NpmRemapResolver.ts b/packages/plugin-npm/sources/NpmRemapResolver.ts index 2746db6b88c3..76f8dcd4cfac 100644 --- a/packages/plugin-npm/sources/NpmRemapResolver.ts +++ b/packages/plugin-npm/sources/NpmRemapResolver.ts @@ -40,10 +40,10 @@ export class NpmRemapResolver implements Resolver { return await opts.resolver.getCandidates(nextDescriptor, dependencies, opts); } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { const nextDescriptor = structUtils.parseDescriptor(descriptor.range.slice(PROTOCOL.length), true); - return opts.resolver.getSatisfying(nextDescriptor, references, dependencies, opts); + return opts.resolver.getSatisfying(nextDescriptor, references, opts); } resolve(locator: Locator, opts: ResolveOptions): never { diff --git a/packages/plugin-npm/sources/NpmSemverResolver.ts b/packages/plugin-npm/sources/NpmSemverResolver.ts index ea08024b39e0..000caf74632b 100644 --- a/packages/plugin-npm/sources/NpmSemverResolver.ts +++ b/packages/plugin-npm/sources/NpmSemverResolver.ts @@ -82,7 +82,7 @@ export class NpmSemverResolver implements Resolver { }); } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { const range = descriptor.range.slice(PROTOCOL.length); const versions = references .map(reference => reference.slice(PROTOCOL.length)) diff --git a/packages/plugin-npm/sources/NpmTagResolver.ts b/packages/plugin-npm/sources/NpmTagResolver.ts index 171d2e0c9ec3..820ab255b91f 100644 --- a/packages/plugin-npm/sources/NpmTagResolver.ts +++ b/packages/plugin-npm/sources/NpmTagResolver.ts @@ -64,7 +64,7 @@ export class NpmTagResolver implements Resolver { } } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { // We can't statically know if a tag resolves to a specific version without using the network return null; } diff --git a/packages/plugin-patch/sources/PatchResolver.ts b/packages/plugin-patch/sources/PatchResolver.ts index 326a39f995d1..d15e47c3d273 100644 --- a/packages/plugin-patch/sources/PatchResolver.ts +++ b/packages/plugin-patch/sources/PatchResolver.ts @@ -61,7 +61,7 @@ export class PatchResolver implements Resolver { return [patchUtils.makeLocator(descriptor, {parentLocator, sourcePackage, patchPaths, patchHash})]; } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { return null; } diff --git a/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts b/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts index cc2bfdf8aae6..6b0726675362 100644 --- a/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts +++ b/packages/yarnpkg-core/sources/LegacyMigrationResolver.ts @@ -7,7 +7,7 @@ import {Project} from './Project'; import {Report} from './Report'; import {Resolver, ResolveOptions, MinimalResolveOptions} from './Resolver'; import * as structUtils from './structUtils'; -import {DescriptorHash, Descriptor, Locator, Package} from './types'; +import {DescriptorHash, Descriptor, Locator} from './types'; const IMPORTED_PATTERNS: Array<[RegExp, (version: string, ...args: Array) => string]> = [ // These ones come from Git urls @@ -115,7 +115,7 @@ export class LegacyMigrationResolver implements Resolver { return [resolution]; } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { return null; } diff --git a/packages/yarnpkg-core/sources/LockfileResolver.ts b/packages/yarnpkg-core/sources/LockfileResolver.ts index fa8755c276dd..f8a61fb4960e 100644 --- a/packages/yarnpkg-core/sources/LockfileResolver.ts +++ b/packages/yarnpkg-core/sources/LockfileResolver.ts @@ -1,6 +1,6 @@ import {Resolver, ResolveOptions, MinimalResolveOptions} from './Resolver'; import * as structUtils from './structUtils'; -import {Descriptor, Locator, DescriptorHash, Package} from './types'; +import {Descriptor, Locator} from './types'; export class LockfileResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { @@ -51,7 +51,7 @@ export class LockfileResolver implements Resolver { return [pkg]; } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { return null; } diff --git a/packages/yarnpkg-core/sources/MultiResolver.ts b/packages/yarnpkg-core/sources/MultiResolver.ts index 411b7d568670..b382eaff7691 100644 --- a/packages/yarnpkg-core/sources/MultiResolver.ts +++ b/packages/yarnpkg-core/sources/MultiResolver.ts @@ -45,10 +45,10 @@ export class MultiResolver implements Resolver { return await resolver.getCandidates(descriptor, dependencies, opts); } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { const resolver = this.getResolverByDescriptor(descriptor, opts); - return resolver.getSatisfying(descriptor, references, dependencies, opts); + return resolver.getSatisfying(descriptor, references, opts); } async resolve(locator: Locator, opts: ResolveOptions) { diff --git a/packages/yarnpkg-core/sources/ProtocolResolver.ts b/packages/yarnpkg-core/sources/ProtocolResolver.ts index 029406d79eb2..714611139631 100644 --- a/packages/yarnpkg-core/sources/ProtocolResolver.ts +++ b/packages/yarnpkg-core/sources/ProtocolResolver.ts @@ -43,8 +43,8 @@ export class ProtocolResolver implements Resolver { return await opts.resolver.getCandidates(this.forwardDescriptor(descriptor, opts), dependencies, opts); } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { - return await opts.resolver.getSatisfying(this.forwardDescriptor(descriptor, opts), references, dependencies, opts); + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { + return await opts.resolver.getSatisfying(this.forwardDescriptor(descriptor, opts), references, opts); } async resolve(locator: Locator, opts: ResolveOptions) { diff --git a/packages/yarnpkg-core/sources/Resolver.ts b/packages/yarnpkg-core/sources/Resolver.ts index 0cf91ca29bba..c0d9173bb802 100644 --- a/packages/yarnpkg-core/sources/Resolver.ts +++ b/packages/yarnpkg-core/sources/Resolver.ts @@ -143,10 +143,9 @@ export interface Resolver { * * @param descriptor The target descriptor. * @param references The candidate references. - * @param dependencies The resolution dependencies and their resolutions. * @param opts The resolution options. */ - getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions): Promise | null>; + getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions): Promise | null>; /** * This function will, given a locator, return the full package definition diff --git a/packages/yarnpkg-core/sources/RunInstallPleaseResolver.ts b/packages/yarnpkg-core/sources/RunInstallPleaseResolver.ts index 140ac1a477a5..5b5254a36a20 100644 --- a/packages/yarnpkg-core/sources/RunInstallPleaseResolver.ts +++ b/packages/yarnpkg-core/sources/RunInstallPleaseResolver.ts @@ -34,7 +34,7 @@ export class RunInstallPleaseResolver implements Resolver { throw new ReportError(MessageName.MISSING_LOCKFILE_ENTRY, `This package doesn't seem to be present in your lockfile; try to make an install to update your resolutions`); } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: unknown, opts: ResolveOptions): Promise { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions): Promise { throw new ReportError(MessageName.MISSING_LOCKFILE_ENTRY, `This package doesn't seem to be present in your lockfile; try to make an install to update your resolutions`); } diff --git a/packages/yarnpkg-core/sources/VirtualResolver.ts b/packages/yarnpkg-core/sources/VirtualResolver.ts index 48e9584f9a5f..6c0f757bc8bf 100644 --- a/packages/yarnpkg-core/sources/VirtualResolver.ts +++ b/packages/yarnpkg-core/sources/VirtualResolver.ts @@ -1,5 +1,5 @@ import {Resolver, ResolveOptions, MinimalResolveOptions} from './Resolver'; -import {Descriptor, Locator, DescriptorHash, Package} from './types'; +import {Descriptor, Locator} from './types'; export class VirtualResolver implements Resolver { static protocol = `virtual:`; @@ -54,7 +54,7 @@ export class VirtualResolver implements Resolver { throw new Error(`Assertion failed: calling "getCandidates" on a virtual descriptor is unsupported`); } - async getSatisfying(descriptor: Descriptor, candidates: Array, dependencies: Map, opts: ResolveOptions): Promise { + async getSatisfying(descriptor: Descriptor, candidates: Array, opts: ResolveOptions): Promise { // It's unsupported because packages inside the dependency tree should // only become virtual AFTER they have all been resolved, by which point // you shouldn't need to call `getSatisfying` anymore. diff --git a/packages/yarnpkg-core/sources/WorkspaceResolver.ts b/packages/yarnpkg-core/sources/WorkspaceResolver.ts index edb0231218f9..47a6dfdf34f0 100644 --- a/packages/yarnpkg-core/sources/WorkspaceResolver.ts +++ b/packages/yarnpkg-core/sources/WorkspaceResolver.ts @@ -1,8 +1,7 @@ import {PortablePath} from '@yarnpkg/fslib'; import {Resolver, ResolveOptions, MinimalResolveOptions} from './Resolver'; -import * as structUtils from './structUtils'; -import {Descriptor, Locator, DescriptorHash, Package} from './types'; +import {Descriptor, Locator} from './types'; import {LinkType} from './types'; export class WorkspaceResolver implements Resolver { @@ -44,7 +43,7 @@ export class WorkspaceResolver implements Resolver { return [workspace.anchoredLocator]; } - async getSatisfying(descriptor: Descriptor, references: Array, dependencies: Map, opts: ResolveOptions) { + async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { return null; } From 24659868b86e6178ad5b06dca8907804d87c2088 Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sun, 23 Aug 2020 13:49:29 +0300 Subject: [PATCH 21/24] chore: lint --- packages/plugin-npm/sources/NpmTagResolver.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/plugin-npm/sources/NpmTagResolver.ts b/packages/plugin-npm/sources/NpmTagResolver.ts index 820ab255b91f..47436303d311 100644 --- a/packages/plugin-npm/sources/NpmTagResolver.ts +++ b/packages/plugin-npm/sources/NpmTagResolver.ts @@ -1,10 +1,10 @@ -import {ReportError, MessageName, Resolver, ResolveOptions, MinimalResolveOptions, TAG_REGEXP, DescriptorHash} from '@yarnpkg/core'; -import {structUtils} from '@yarnpkg/core'; -import {Descriptor, Locator, Package} from '@yarnpkg/core'; +import {ReportError, MessageName, Resolver, ResolveOptions, MinimalResolveOptions, TAG_REGEXP} from '@yarnpkg/core'; +import {structUtils} from '@yarnpkg/core'; +import {Descriptor, Locator, Package} from '@yarnpkg/core'; -import {NpmSemverFetcher} from './NpmSemverFetcher'; -import {PROTOCOL} from './constants'; -import * as npmHttpUtils from './npmHttpUtils'; +import {NpmSemverFetcher} from './NpmSemverFetcher'; +import {PROTOCOL} from './constants'; +import * as npmHttpUtils from './npmHttpUtils'; export class NpmTagResolver implements Resolver { supportsDescriptor(descriptor: Descriptor, opts: MinimalResolveOptions) { From 1fc26fa0f652c3cdf4e000a0727d2794f230877c Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sun, 23 Aug 2020 13:58:15 +0300 Subject: [PATCH 22/24] refactor: skip -> assertion failed --- packages/plugin-essentials/sources/commands/dedupe.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/plugin-essentials/sources/commands/dedupe.ts b/packages/plugin-essentials/sources/commands/dedupe.ts index 32f3c481408e..432f85b073a7 100644 --- a/packages/plugin-essentials/sources/commands/dedupe.ts +++ b/packages/plugin-essentials/sources/commands/dedupe.ts @@ -77,7 +77,7 @@ export const DEDUPE_ALGORITHMS: Record = { const locators = locatorsByIdent.get(descriptor.identHash); if (typeof locators === `undefined`) - return null; + throw new Error(`Assertion failed: The resolutions (${descriptor.identHash}) should have been registered`); // No need to choose when there's only one possibility if (locators.size === 1) @@ -145,7 +145,7 @@ export default class DedupeCommand extends BaseCommand { - \`highest\`: Reuses (where possible) the locators with the highest versions. This means that dependencies can only be upgraded, never downgraded. It's also guaranteed that it never takes more than a single pass to dedupe the entire dependency tree. - **Note:** Even though it never produces a wrong dependency tree, this command should be used with caution, as it modifies the dependency tree, which can sometimes cause problems when packages strictly follow semver recommandations. It is recommended to also review the changes manually. + **Note:** Even though it never produces a wrong dependency tree, this command should be used with caution, as it modifies the dependency tree, which can sometimes cause problems when packages don't strictly follow semver recommendations. Because of this, it is recommended to also review the changes manually. If set, the \`-c,--check\` flag will only report the found duplicates, without persisting the modified dependency tree. If changes are found, the command will exit with a non-zero exit code, making it suitable for CI purposes. From d05b28446fd4928e17dcd50e403d160f5b9fe5dd Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sun, 23 Aug 2020 15:33:10 +0300 Subject: [PATCH 23/24] Apply suggestions from code review Co-authored-by: Kristoffer K. --- packages/plugin-npm/sources/NpmSemverResolver.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/plugin-npm/sources/NpmSemverResolver.ts b/packages/plugin-npm/sources/NpmSemverResolver.ts index 000caf74632b..6bcf46493ed9 100644 --- a/packages/plugin-npm/sources/NpmSemverResolver.ts +++ b/packages/plugin-npm/sources/NpmSemverResolver.ts @@ -83,13 +83,13 @@ export class NpmSemverResolver implements Resolver { } async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { - const range = descriptor.range.slice(PROTOCOL.length); + const range = new semver.Range(descriptor.range.slice(PROTOCOL.length)); const versions = references .map(reference => reference.slice(PROTOCOL.length)) .filter(reference => semver.valid(reference)); return versions - .filter(version => semver.satisfies(version, range)) + .filter(version => range.test(version)) .sort((a, b) => -semver.compare(a, b)) .map(version => structUtils.makeLocator(descriptor, `${PROTOCOL}${version}`)); } From 50f0c5172be577e5e0c1ebde91d5da00ff6a375c Mon Sep 17 00:00:00 2001 From: Paul Soporan Date: Sun, 23 Aug 2020 16:24:40 +0300 Subject: [PATCH 24/24] refactor: use semver.SemVer instances Co-authored-by: Kristoffer K. --- .../plugin-npm/sources/NpmSemverResolver.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/plugin-npm/sources/NpmSemverResolver.ts b/packages/plugin-npm/sources/NpmSemverResolver.ts index 6bcf46493ed9..ba49aa5932ac 100644 --- a/packages/plugin-npm/sources/NpmSemverResolver.ts +++ b/packages/plugin-npm/sources/NpmSemverResolver.ts @@ -84,14 +84,19 @@ export class NpmSemverResolver implements Resolver { async getSatisfying(descriptor: Descriptor, references: Array, opts: ResolveOptions) { const range = new semver.Range(descriptor.range.slice(PROTOCOL.length)); - const versions = references - .map(reference => reference.slice(PROTOCOL.length)) - .filter(reference => semver.valid(reference)); - return versions + return references + .map(reference => { + try { + return new semver.SemVer(reference.slice(PROTOCOL.length)); + } catch { + return null; + } + }) + .filter((version): version is semver.SemVer => version !== null) .filter(version => range.test(version)) - .sort((a, b) => -semver.compare(a, b)) - .map(version => structUtils.makeLocator(descriptor, `${PROTOCOL}${version}`)); + .sort((a, b) => -a.compare(b)) + .map(version => structUtils.makeLocator(descriptor, `${PROTOCOL}${version.raw}`)); } async resolve(locator: Locator, opts: ResolveOptions) {