feat(nodejs): add v9 pnpm lock file support #6617
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -2,10 +2,12 @@ package pnpm | |||||
|
|
||||||
| import ( | ||||||
| "fmt" | ||||||
| "sort" | ||||||
| "strconv" | ||||||
| "strings" | ||||||
|
|
||||||
| "github.com/samber/lo" | ||||||
| "golang.org/x/exp/maps" | ||||||
|
There was a problem hiding this comment.
Suggested change
There was a problem hiding this comment. This is weird, but |
||||||
| "golang.org/x/xerrors" | ||||||
| "gopkg.in/yaml.v3" | ||||||
|
|
||||||
|
|
@@ -34,6 +36,28 @@ type LockFile struct { | |||||
| Dependencies map[string]any `yaml:"dependencies,omitempty"` | ||||||
| DevDependencies map[string]any `yaml:"devDependencies,omitempty"` | ||||||
| Packages map[string]PackageInfo `yaml:"packages,omitempty"` | ||||||
|
|
||||||
| // V9 | ||||||
| Importers Importer `yaml:"importers,omitempty"` | ||||||
| Snapshots map[string]Snapshot `yaml:"snapshots,omitempty"` | ||||||
| } | ||||||
|
|
||||||
| type Importer struct { | ||||||
| Root RootImporter `yaml:".,omitempty"` | ||||||
| } | ||||||
|
|
||||||
| type RootImporter struct { | ||||||
| Dependencies map[string]ImporterDepVersion `yaml:"dependencies,omitempty"` | ||||||
| DevDependencies map[string]ImporterDepVersion `yaml:"devDependencies,omitempty"` | ||||||
| } | ||||||
|
|
||||||
| type ImporterDepVersion struct { | ||||||
| Version string `yaml:"version,omitempty"` | ||||||
| } | ||||||
|
|
||||||
| type Snapshot struct { | ||||||
| Dependencies map[string]string `yaml:"dependencies,omitempty"` | ||||||
| OptionalDependencies map[string]string `yaml:"optionalDependencies,omitempty"` | ||||||
| } | ||||||
|
|
||||||
| type Parser struct { | ||||||
|
|
@@ -57,8 +81,16 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc | |||||
| return nil, nil, nil | ||||||
| } | ||||||
|
|
||||||
| var pkgs []ftypes.Package | ||||||
| var deps []ftypes.Dependency | ||||||
| if lockVer >= 9 { | ||||||
| pkgs, deps = p.parseV9(lockFile) | ||||||
| } else { | ||||||
| pkgs, deps = p.parse(lockVer, lockFile) | ||||||
| } | ||||||
|
|
||||||
| sort.Sort(ftypes.Packages(pkgs)) | ||||||
| sort.Sort(ftypes.Dependencies(deps)) | ||||||
| return pkgs, deps, nil | ||||||
| } | ||||||
|
|
||||||
|
|
@@ -78,9 +110,11 @@ func (p *Parser) parse(lockVer float64, lockFile LockFile) ([]ftypes.Package, [] | |||||
| // cf. https://github.com/pnpm/spec/blob/274ff02de23376ad59773a9f25ecfedd03a41f64/lockfile/6.0.md#packagesdependencypathname | ||||||
| name := info.Name | ||||||
| version := info.Version | ||||||
| var ref string | ||||||
|
|
||||||
| if name == "" { | ||||||
| name, version, ref = p.parseDepPath(depPath, lockVer) | ||||||
| version = p.parseVersion(depPath, version, lockVer) | ||||||
| } | ||||||
| pkgID := packageID(name, version) | ||||||
|
|
||||||
|
|
@@ -90,13 +124,15 @@ func (p *Parser) parse(lockVer float64, lockFile LockFile) ([]ftypes.Package, [] | |||||
| } | ||||||
|
|
||||||
| pkgs = append(pkgs, ftypes.Package{ | ||||||
| ID: pkgID, | ||||||
| Name: name, | ||||||
| Version: version, | ||||||
| Relationship: lo.Ternary(isDirectPkg(name, lockFile.Dependencies), ftypes.RelationshipDirect, ftypes.RelationshipIndirect), | ||||||
| ExternalReferences: toExternalRefs(ref), | ||||||
| }) | ||||||
|
|
||||||
| if len(dependencies) > 0 { | ||||||
| sort.Strings(dependencies) | ||||||
| deps = append(deps, ftypes.Dependency{ | ||||||
| ID: pkgID, | ||||||
| DependsOn: dependencies, | ||||||
|
|
@@ -107,6 +143,98 @@ func (p *Parser) parse(lockVer float64, lockFile LockFile) ([]ftypes.Package, [] | |||||
| return pkgs, deps | ||||||
| } | ||||||
|
|
||||||
| func (p *Parser) parseV9(lockFile LockFile) ([]ftypes.Package, []ftypes.Dependency) { | ||||||
| lockVer := 9.0 | ||||||
| resolvedPkgs := make(map[string]ftypes.Package) | ||||||
| resolvedDeps := make(map[string]ftypes.Dependency) | ||||||
|
|
||||||
| // Check all snapshots and save with resolved versions | ||||||
| resolvedSnapshots := make(map[string][]string) | ||||||
| for depPath, snapshot := range lockFile.Snapshots { | ||||||
| name, version, _ := p.parseDepPath(depPath, lockVer) | ||||||
|
|
||||||
| var dependsOn []string | ||||||
| for depName, depVer := range lo.Assign(snapshot.OptionalDependencies, snapshot.Dependencies) { | ||||||
| depVer = p.trimPeerDeps(depVer, lockVer) // pnpm has already separated dep name. therefore, we only need to separate peer deps. | ||||||
| depVer = p.parseVersion(depPath, depVer, lockVer) | ||||||
| id := packageID(depName, depVer) | ||||||
| if _, ok := lockFile.Packages[id]; ok { | ||||||
| dependsOn = append(dependsOn, id) | ||||||
| } | ||||||
| } | ||||||
| if len(dependsOn) > 0 { | ||||||
| resolvedSnapshots[packageID(name, version)] = dependsOn | ||||||
| } | ||||||
|
|
||||||
| } | ||||||
|
|
||||||
| for depPath, pkgInfo := range lockFile.Packages { | ||||||
| name, ver, ref := p.parseDepPath(depPath, lockVer) | ||||||
| parsedVer := p.parseVersion(depPath, ver, lockVer) | ||||||
|
|
||||||
| if pkgInfo.Version != "" { | ||||||
| parsedVer = pkgInfo.Version | ||||||
| } | ||||||
|
|
||||||
| // By default, pkg is dev pkg. | ||||||
| // We will update `Dev` field later. | ||||||
| dev := true | ||||||
| relationship := ftypes.RelationshipIndirect | ||||||
| if dep, ok := lockFile.Importers.Root.DevDependencies[name]; ok && dep.Version == ver { | ||||||
| relationship = ftypes.RelationshipDirect | ||||||
| } | ||||||
| if dep, ok := lockFile.Importers.Root.Dependencies[name]; ok && dep.Version == ver { | ||||||
| relationship = ftypes.RelationshipDirect | ||||||
| dev = false // mark root direct deps to update `dev` field of their child deps. | ||||||
| } | ||||||
|
|
||||||
| id := packageID(name, parsedVer) | ||||||
| resolvedPkgs[id] = ftypes.Package{ | ||||||
| ID: id, | ||||||
| Name: name, | ||||||
| Version: parsedVer, | ||||||
| Relationship: relationship, | ||||||
| Dev: dev, | ||||||
| ExternalReferences: toExternalRefs(ref), | ||||||
| } | ||||||
|
|
||||||
| // Save child deps | ||||||
| if dependsOn, ok := resolvedSnapshots[depPath]; ok { | ||||||
| sort.Strings(dependsOn) | ||||||
| resolvedDeps[id] = ftypes.Dependency{ | ||||||
| ID: id, | ||||||
| DependsOn: dependsOn, // Deps from dependsOn has been resolved when parsing snapshots | ||||||
| } | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| // Overwrite the `Dev` field for dev deps and their child dependencies. | ||||||
| for _, pkg := range resolvedPkgs { | ||||||
| if !pkg.Dev { | ||||||
| p.markRootPkgs(pkg.ID, resolvedPkgs, resolvedDeps) | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| return maps.Values(resolvedPkgs), maps.Values(resolvedDeps) | ||||||
| } | ||||||
|
|
||||||
| // markRootPkgs sets `Dev` to false for non dev dependency. | ||||||
| func (p *Parser) markRootPkgs(id string, pkgs map[string]ftypes.Package, deps map[string]ftypes.Dependency) { | ||||||
| pkg, ok := pkgs[id] | ||||||
| if !ok { | ||||||
| return | ||||||
| } | ||||||
|
|
||||||
| pkg.Dev = false | ||||||
| pkgs[id] = pkg | ||||||
|
|
||||||
| // Update child deps | ||||||
| for _, depID := range deps[id].DependsOn { | ||||||
| p.markRootPkgs(depID, pkgs, deps) | ||||||
| } | ||||||
| return | ||||||
| } | ||||||
|
|
||||||
| func (p *Parser) parseLockfileVersion(lockFile LockFile) float64 { | ||||||
| switch v := lockFile.LockfileVersion.(type) { | ||||||
| // v5 | ||||||
|
|
@@ -127,55 +255,109 @@ func (p *Parser) parseLockfileVersion(lockFile LockFile) float64 { | |||||
| } | ||||||
| } | ||||||
|
|
||||||
| func (p *Parser) parseDepPath(depPath string, lockVer float64) (string, string, string) { | ||||||
| dPath, nonDefaultRegistry := p.trimRegistry(depPath, lockVer) | ||||||
|
|
||||||
| var scope string | ||||||
| scope, dPath = p.separateScope(dPath) | ||||||
|
|
||||||
| var name string | ||||||
| name, dPath = p.separateName(dPath, lockVer) | ||||||
|
|
||||||
| // add scope to pkg name | ||||||
| if scope != "" { | ||||||
| name = fmt.Sprintf("%s/%s", scope, name) | ||||||
| } | ||||||
|
|
||||||
| ver := p.trimPeerDeps(dPath, lockVer) | ||||||
|
|
||||||
| return name, ver, lo.Ternary(nonDefaultRegistry, depPath, "") | ||||||
| } | ||||||
|
|
||||||
| // trimRegistry trims registry (or `/` prefix) for depPath. | ||||||
|
There was a problem hiding this comment. PURL has There was a problem hiding this comment. I am not sure if using registry URL in package name is correct. Lines 425 to 429 in bbaf595 e.g. ID: "@babel/runtime@7.18.3",
Name: "@babel/runtime",
Version: "7.18.3",
Relationship: ftypes.RelationshipIndirect,
ExternalReferences: []ftypes.ExternalRef{
{
Type: ftypes.RefVCS,
URL: "private.npmjs.org/@babel/runtime@7.18.3",
},
},@knqyf263 wdyt? There was a problem hiding this comment. Yes, I meant to suggest using There was a problem hiding this comment. I update this logic - 2d50522 |
||||||
| // It returns true if non-default registry has been trimmed. | ||||||
| // e.g. | ||||||
| // - "registry.npmjs.org/lodash/4.17.10" => "lodash/4.17.10", false | ||||||
| // - "registry.npmjs.org/@babel/generator/7.21.9" => "@babel/generator/7.21.9", false | ||||||
| // - "private.npm.org/@babel/generator/7.21.9" => "@babel/generator/7.21.9", true | ||||||
| // - "/lodash/4.17.10" => "lodash/4.17.10", false | ||||||
| // - "/asap@2.0.6" => "asap@2.0.6", false | ||||||
| func (p *Parser) trimRegistry(depPath string, lockVer float64) (string, bool) { | ||||||
| var nonDefaultRegistry bool | ||||||
| // lock file v9 doesn't use registry prefix | ||||||
| if lockVer < 9 { | ||||||
| var registry string | ||||||
| registry, depPath, _ = strings.Cut(depPath, "/") | ||||||
| if registry != "" && registry != "registry.npmjs.org" { | ||||||
| nonDefaultRegistry = true | ||||||
| } | ||||||
| } | ||||||
| return depPath, nonDefaultRegistry | ||||||
| } | ||||||
|
|
||||||
| // separateScope separates the scope (if set) from the rest of the depPath. | ||||||
| // e.g. | ||||||
| // - v5: "@babel/generator/7.21.9" => {"babel", "generator/7.21.9"} | ||||||
| // - v6+: "@babel/helper-annotate-as-pure@7.18.6" => "{"babel", "helper-annotate-as-pure@7.18.6"} | ||||||
| func (p *Parser) separateScope(depPath string) (string, string) { | ||||||
| var scope string | ||||||
| if strings.HasPrefix(depPath, "@") { | ||||||
| scope, depPath, _ = strings.Cut(depPath, "/") | ||||||
| } | ||||||
| return scope, depPath | ||||||
| } | ||||||
|
|
||||||
| // separateName separates pkg name and version. | ||||||
| // e.g. | ||||||
| // - v5: "generator/7.21.9" => {"generator", "7.21.9"} | ||||||
| // - v6+: "7.21.5(@babel/core@7.20.7)" => "7.21.5" | ||||||
| // | ||||||
| // for v9+ version can be filePath or link: | ||||||
| // - "package1@file:package1:" | ||||||
| // - "is-negative@https://codeload.github.com/zkochan/is-negative/tar.gz/2fa0531ab04e300a24ef4fd7fb3a280eccb7ccc5" | ||||||
| // | ||||||
| // Also version can contain peer deps: | ||||||
| // - "debug@4.3.4(supports-color@8.1.1)" | ||||||
| func (p *Parser) separateName(depPath string, lockVer float64) (string, string) { | ||||||
| sep := "@" | ||||||
| if lockVer < 6 { | ||||||
| sep = "/" | ||||||
| } | ||||||
| name, version, _ := strings.Cut(depPath, sep) | ||||||
| return name, version | ||||||
| } | ||||||
|
|
||||||
| // Trim peer deps | ||||||
| // e.g. | ||||||
| // - v5: "7.21.5_@babel+core@7.21.8" => "7.21.5" | ||||||
| // - v6+: "7.21.5(@babel/core@7.20.7)" => "7.21.5" | ||||||
| func (p *Parser) trimPeerDeps(depPath string, lockVer float64) string { | ||||||
| sep := "(" | ||||||
| if lockVer < 6 { | ||||||
| sep = "_" | ||||||
| } | ||||||
| version, _, _ := strings.Cut(depPath, sep) | ||||||
| return version | ||||||
| } | ||||||
|
|
||||||
| // parseVersion parses version. | ||||||
| // v9 can use filePath or link as version - we need to clear these versions. | ||||||
| // e.g. | ||||||
| // - "package1@file:package1:" | ||||||
| // - "is-negative@https://codeload.github.com/zkochan/is-negative/tar.gz/2fa0531ab04e300a24ef4fd7fb3a280eccb7ccc5" | ||||||
| // | ||||||
| // Other versions should be semver valid. | ||||||
| func (p *Parser) parseVersion(depPath, ver string, lockVer float64) string { | ||||||
| if lockVer < 9 && (strings.HasPrefix(ver, "file:") || strings.HasPrefix(ver, "http")) { | ||||||
| return "" | ||||||
| } | ||||||
| if _, err := semver.Parse(ver); err != nil { | ||||||
| p.logger.Debug("Skip non-semver package", log.String("pkg_path", depPath), | ||||||
| log.String("version", ver), log.Err(err)) | ||||||
| return "" | ||||||
| } | ||||||
|
|
||||||
| return ver | ||||||
| } | ||||||
|
|
||||||
| func isDirectPkg(name string, directDeps map[string]interface{}) bool { | ||||||
|
|
@@ -186,3 +368,15 @@ func isDirectPkg(name string, directDeps map[string]interface{}) bool { | |||||
| func packageID(name, version string) string { | ||||||
| return dependency.ID(ftypes.Pnpm, name, version) | ||||||
| } | ||||||
|
|
||||||
| func toExternalRefs(ref string) []ftypes.ExternalRef { | ||||||
| if ref == "" { | ||||||
| return nil | ||||||
| } | ||||||
| return []ftypes.ExternalRef{ | ||||||
| { | ||||||
| Type: ftypes.RefVCS, | ||||||
| URL: ref, | ||||||
| }, | ||||||
| } | ||||||
| } | ||||||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I have not seen any requests to use
Devfield for version6.0or earlier lock files.But if users want, we can add this in another PR.