Skip to content

Commit

Permalink
fix: further improvements to CPE generation for apk packages (#1623)
Browse files Browse the repository at this point in the history 
* fix: consider upstream logic during apk cpe gen
* fix: correct apk CPE for go
* fix: correct apk CPE for ruby
* fix: correct apk CPE for bazel
* fix: correct apk CPE for clang
* fix: correct apk CPE for openjdk
* fix: correct apk CPE for glibc
* fix: correct apk CPE for gli
* fix: correct apk CPE for bas
* fix: correct apk CPE for alsa-lib
* fix: correct apk CPE for alsa
* fix: determine apk cpe vendor from known URLs
* fix: add more url prefix->vendor mappings for apk
* refactor: allow reuse of vendor by url prefix logic
* feat: extract username as vendor candidate from github/gitlab

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>
  • Loading branch information
@westonsteimel
westonsteimel authored Feb 27, 2023
1 parent d23b4d4 commit fbda21f
Show file tree
Hide file tree
Showing 5 changed files with 329 additions and 60 deletions.
125 changes: 73 additions & 52 deletions syft/pkg/cataloger/common/cpe/apk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,12 @@ var (

func pythonCandidateVendorsFromName(v string) fieldCandidateSet {
vendors := newFieldCandidateSet()
vendors.add(fieldCandidate{
value: v,
disallowSubSelections: true,
disallowDelimiterVariations: true,
})

vendors.addValue(v)
vendors.addValue(findAdditionalVendors(defaultCandidateAdditions, pkg.PythonPkg, v, v)...)
vendors.removeByValue(findVendorsToRemove(defaultCandidateRemovals, pkg.PythonPkg, v)...)

for _, av := range additionalVendorsForPython(v) {
vendors.add(fieldCandidate{
value: av,
disallowSubSelections: true,
disallowDelimiterVariations: true,
})
vendors.addValue(av)
vendors.addValue(findAdditionalVendors(defaultCandidateAdditions, pkg.PythonPkg, av, av)...)
vendors.removeByValue(findVendorsToRemove(defaultCandidateRemovals, pkg.PythonPkg, av)...)
}
Expand All @@ -37,15 +28,16 @@ func pythonCandidateVendorsFromName(v string) fieldCandidateSet {

func pythonCandidateVendorsFromAPK(m pkg.ApkMetadata) fieldCandidateSet {
vendors := newFieldCandidateSet()
upstream := m.Upstream()

for _, p := range pythonPrefixes {
if strings.HasPrefix(m.Package, p) {
t := strings.ToLower(strings.TrimPrefix(m.Package, p))
vendors.union(pythonCandidateVendorsFromName(t))
}

if m.OriginPackage != m.Package && strings.HasPrefix(m.OriginPackage, p) {
t := strings.ToLower(strings.TrimPrefix(m.OriginPackage, p))
if upstream != m.Package && strings.HasPrefix(upstream, p) {
t := strings.ToLower(strings.TrimPrefix(upstream, p))
vendors.union(pythonCandidateVendorsFromName(t))
}
}
Expand All @@ -55,28 +47,24 @@ func pythonCandidateVendorsFromAPK(m pkg.ApkMetadata) fieldCandidateSet {

func pythonCandidateProductsFromName(p string) fieldCandidateSet {
products := newFieldCandidateSet()
products.add(fieldCandidate{
value: p,
disallowSubSelections: true,
disallowDelimiterVariations: true,
})

products.addValue(p)
products.addValue(findAdditionalProducts(defaultCandidateAdditions, pkg.PythonPkg, p)...)
products.removeByValue(findProductsToRemove(defaultCandidateRemovals, pkg.PythonPkg, p)...)
return products
}

func pythonCandidateProductsFromAPK(m pkg.ApkMetadata) fieldCandidateSet {
products := newFieldCandidateSet()
upstream := m.Upstream()

for _, p := range pythonPrefixes {
if strings.HasPrefix(m.Package, p) {
t := strings.ToLower(strings.TrimPrefix(m.Package, p))
products.union(pythonCandidateProductsFromName(t))
}

if m.OriginPackage != m.Package && strings.HasPrefix(m.OriginPackage, p) {
t := strings.ToLower(strings.TrimPrefix(m.OriginPackage, p))
if upstream != m.Package && strings.HasPrefix(upstream, p) {
t := strings.ToLower(strings.TrimPrefix(upstream, p))
products.union(pythonCandidateProductsFromName(t))
}
}
Expand All @@ -86,29 +74,27 @@ func pythonCandidateProductsFromAPK(m pkg.ApkMetadata) fieldCandidateSet {

func rubyCandidateVendorsFromName(v string) fieldCandidateSet {
vendors := newFieldCandidateSet()
vendors.add(fieldCandidate{
value: v,
disallowSubSelections: true,
disallowDelimiterVariations: true,
})

vendors.addValue(v)
vendors.addValue(findAdditionalVendors(defaultCandidateAdditions, pkg.GemPkg, v, v)...)
vendors.removeByValue(findVendorsToRemove(defaultCandidateRemovals, pkg.GemPkg, v)...)
return vendors
}

func rubyCandidateVendorsFromAPK(m pkg.ApkMetadata) fieldCandidateSet {
vendors := newFieldCandidateSet()

for _, p := range rubyPrefixes {
if strings.HasPrefix(m.Package, p) {
t := strings.ToLower(strings.TrimPrefix(m.Package, p))
vendors.union(rubyCandidateVendorsFromName(t))
}

if m.OriginPackage != m.Package && strings.HasPrefix(m.OriginPackage, p) {
t := strings.ToLower(strings.TrimPrefix(m.OriginPackage, p))
vendors.union(rubyCandidateVendorsFromName(t))
upstream := m.Upstream()

if upstream != "ruby" {
for _, p := range rubyPrefixes {
if strings.HasPrefix(m.Package, p) {
t := strings.ToLower(strings.TrimPrefix(m.Package, p))
vendors.union(rubyCandidateVendorsFromName(t))
}

if upstream != "" && upstream != m.Package && strings.HasPrefix(upstream, p) {
t := strings.ToLower(strings.TrimPrefix(upstream, p))
vendors.union(rubyCandidateVendorsFromName(t))
}
}
}

Expand All @@ -117,30 +103,52 @@ func rubyCandidateVendorsFromAPK(m pkg.ApkMetadata) fieldCandidateSet {

func rubyCandidateProductsFromName(p string) fieldCandidateSet {
products := newFieldCandidateSet()
products.add(fieldCandidate{
value: p,
disallowSubSelections: true,
disallowDelimiterVariations: true,
})

products.addValue(p)
products.addValue(findAdditionalProducts(defaultCandidateAdditions, pkg.GemPkg, p)...)
products.removeByValue(findProductsToRemove(defaultCandidateRemovals, pkg.GemPkg, p)...)
return products
}

func rubyCandidateProductsFromAPK(m pkg.ApkMetadata) fieldCandidateSet {
products := newFieldCandidateSet()

for _, p := range rubyPrefixes {
if strings.HasPrefix(m.Package, p) {
t := strings.ToLower(strings.TrimPrefix(m.Package, p))
products.union(rubyCandidateProductsFromName(t))
upstream := m.Upstream()

if upstream != "ruby" {
for _, p := range rubyPrefixes {
if strings.HasPrefix(m.Package, p) {
t := strings.ToLower(strings.TrimPrefix(m.Package, p))
products.union(rubyCandidateProductsFromName(t))
}

if upstream != "" && upstream != m.Package && strings.HasPrefix(upstream, p) {
t := strings.ToLower(strings.TrimPrefix(upstream, p))
products.union(rubyCandidateProductsFromName(t))
}
}
}

if m.OriginPackage != m.Package && strings.HasPrefix(m.OriginPackage, p) {
t := strings.ToLower(strings.TrimPrefix(m.OriginPackage, p))
products.union(rubyCandidateProductsFromName(t))
}
return products
}

func candidateVendorsFromAPKUpstream(m pkg.ApkMetadata) fieldCandidateSet {
vendors := newFieldCandidateSet()
upstream := m.Upstream()
if upstream != "" && upstream != m.Package {
vendors.addValue(upstream)
vendors.addValue(findAdditionalVendors(defaultCandidateAdditions, pkg.ApkPkg, upstream, upstream)...)
vendors.removeByValue(findVendorsToRemove(defaultCandidateRemovals, pkg.ApkPkg, upstream)...)
}

return vendors
}

func candidateProductsFromAPKUpstream(m pkg.ApkMetadata) fieldCandidateSet {
products := newFieldCandidateSet()
upstream := m.Upstream()
if upstream != "" {
products.addValue(upstream)
products.addValue(findAdditionalProducts(defaultCandidateAdditions, pkg.ApkPkg, upstream)...)
products.removeByValue(findProductsToRemove(defaultCandidateRemovals, pkg.ApkPkg, upstream)...)
}

return products
Expand All @@ -155,6 +163,13 @@ func candidateVendorsForAPK(p pkg.Package) fieldCandidateSet {
vendors := newFieldCandidateSet()
vendors.union(pythonCandidateVendorsFromAPK(metadata))
vendors.union(rubyCandidateVendorsFromAPK(metadata))
vendors.union(candidateVendorsFromAPKUpstream(metadata))
vendors.union(candidateVendorsFromURL(metadata.URL))

for v := range vendors {
v.disallowDelimiterVariations = true
v.disallowSubSelections = true
}

return vendors
}
Expand All @@ -168,6 +183,12 @@ func candidateProductsForAPK(p pkg.Package) fieldCandidateSet {
products := newFieldCandidateSet()
products.union(pythonCandidateProductsFromAPK(metadata))
products.union(rubyCandidateProductsFromAPK(metadata))
products.union(candidateProductsFromAPKUpstream(metadata))

for p := range products {
p.disallowDelimiterVariations = true
p.disallowSubSelections = true
}

return products
}
76 changes: 68 additions & 8 deletions syft/pkg/cataloger/common/cpe/apk_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,14 @@ func Test_candidateVendorsForAPK(t *testing.T) {
expected: []string{"python-cryptography_project", "cryptography", "cryptographyproject", "cryptography_project"},
},
{
name: "py2-pypdf OriginPackage",
name: "py2-pypdf with explicit different origin",
pkg: pkg.Package{
Metadata: pkg.ApkMetadata{
OriginPackage: "py2-pypdf",
Package: "py2-pypdf",
OriginPackage: "abcdefg",
},
},
expected: []string{"pypdf", "pypdfproject", "pypdf_project"},
expected: []string{"pypdf", "pypdfproject", "pypdf_project", "abcdefg"},
},
{
name: "ruby-armadillo Package",
Expand All @@ -41,10 +42,39 @@ func Test_candidateVendorsForAPK(t *testing.T) {
},
expected: []string{"armadillo"},
},
{
name: "python-3.6",
pkg: pkg.Package{
Metadata: pkg.ApkMetadata{
Package: "python-3.6",
},
},
expected: []string{"python", "python_software_foundation"},
},
{
name: "ruby-3.6",
pkg: pkg.Package{
Metadata: pkg.ApkMetadata{
Package: "ruby-3.6",
URL: "https://www.ruby-lang.org/",
},
},
expected: []string{"ruby", "ruby-lang"},
},
{
name: "make",
pkg: pkg.Package{
Metadata: pkg.ApkMetadata{
Package: "make",
URL: "https://www.gnu.org/software/make",
},
},
expected: []string{"gnu"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.ElementsMatch(t, test.expected, candidateVendorsForAPK(test.pkg).values(), "different vendors")
assert.ElementsMatch(t, test.expected, candidateVendorsForAPK(test.pkg).uniqueValues(), "different vendors")
})
}
}
Expand All @@ -65,13 +95,14 @@ func Test_candidateProductsForAPK(t *testing.T) {
expected: []string{"cryptography", "python-cryptography"},
},
{
name: "py2-pypdf OriginPackage",
name: "py2-pypdf with explicit different origin",
pkg: pkg.Package{
Metadata: pkg.ApkMetadata{
OriginPackage: "py2-pypdf",
Package: "py2-pypdf",
OriginPackage: "abcdefg",
},
},
expected: []string{"pypdf"},
expected: []string{"pypdf", "abcdefg"},
},
{
name: "ruby-armadillo Package",
Expand All @@ -82,10 +113,39 @@ func Test_candidateProductsForAPK(t *testing.T) {
},
expected: []string{"armadillo"},
},
{
name: "python-3.6",
pkg: pkg.Package{
Metadata: pkg.ApkMetadata{
Package: "python-3.6",
},
},
expected: []string{"python"},
},
{
name: "ruby-3.6",
pkg: pkg.Package{
Metadata: pkg.ApkMetadata{
Package: "ruby-3.6",
URL: "https://www.ruby-lang.org/",
},
},
expected: []string{"ruby"},
},
{
name: "make",
pkg: pkg.Package{
Metadata: pkg.ApkMetadata{
Package: "make",
URL: "https://www.gnu.org/software/make",
},
},
expected: []string{"make"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.ElementsMatch(t, test.expected, candidateProductsForAPK(test.pkg).values(), "different products")
assert.ElementsMatch(t, test.expected, candidateProductsForAPK(test.pkg).uniqueValues(), "different products")
})
}
}
Loading

0 comments on commit fbda21f

Please sign in to comment.