Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: apk product/vendor generation for old metadata #1635

Merged
merged 1 commit into from
Mar 1, 2023
Merged

fix: apk product/vendor generation for old metadata #1635

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 37 additions & 13 deletions syft/pkg/apk_metadata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,12 @@ import (
const ApkDBGlob = "**/lib/apk/db/installed"

var (
_ FileOwner = (*ApkMetadata)(nil)
prefixes = []string{"py-", "py2-", "py3-", "ruby-"}
upstreamPattern = regexp.MustCompile(`^(?P<upstream>[a-zA-Z][\w-]*?)\-?\d[\d\.]*$`)
_ FileOwner = (*ApkMetadata)(nil)
prefixesToPackageType = map[string]Type{
"py-": PythonPkg,
"ruby-": GemPkg,
}
streamVersionPkgNamePattern = regexp.MustCompile(`^(?P<stream>[a-zA-Z][\w-]*?)(?P<streamVersion>\-?\d[\d\.]*?)($|-(?P<subPackage>[a-zA-Z][\w-]*?)?)$`)
)

// ApkMetadata represents all captured data for a Alpine DB package entry.
Expand Down Expand Up @@ -121,23 +124,44 @@ func (m ApkMetadata) OwnedFiles() (result []string) {
return result
}

func (m ApkMetadata) Upstream() string {
type UpstreamCandidate struct {
Name string
Type Type
}

func (m ApkMetadata) UpstreamCandidates() (candidates []UpstreamCandidate) {
Copy link
Contributor

@wagoodman wagoodman Mar 1, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is more of a preference than anything else -- since UpstreamCandidates (and the former Upstream) is only used for the benefit of CPE generation the more ideal spot is where CPE generation is being done, in syft/pkg/cataloger/common/cpe. This also has the benefit of being unexported so it can't be used in the API.

What are your thoughts about this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I would have liked that; however, it's also needed for the purl.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, I guess I could move it under syft/pkg/cataloger/common/cpe and then import it in apkdb?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would still have to be exported though, right?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I missed the usage in purl processing. I take back my comment then. I think in the future this will be in a better position to refactor: later we'll be moving the CPE generation logic back to where packages are created (near the catalogers) which means that we could remove this from the metadata object at that time.

name := m.Package
if m.OriginPackage != "" && m.OriginPackage != m.Package {
return m.OriginPackage
candidates = append(candidates, UpstreamCandidate{Name: m.OriginPackage, Type: ApkPkg})
}

groups := internal.MatchNamedCaptureGroups(upstreamPattern, m.Package)
groups := internal.MatchNamedCaptureGroups(streamVersionPkgNamePattern, m.Package)
stream, ok := groups["stream"]

upstream, ok := groups["upstream"]
if !ok {
upstream = m.Package
if ok && stream != "" {
sub, ok := groups["subPackage"]

if ok && sub != "" {
name = fmt.Sprintf("%s-%s", stream, sub)
} else {
name = stream
}
}

for _, p := range prefixes {
if strings.HasPrefix(upstream, p) {
return strings.TrimPrefix(upstream, p)
for prefix, typ := range prefixesToPackageType {
if strings.HasPrefix(name, prefix) {
t := strings.TrimPrefix(name, prefix)
if t != "" {
candidates = append(candidates, UpstreamCandidate{Name: t, Type: typ})
return candidates
}
}
}

return upstream
if name != "" {
candidates = append(candidates, UpstreamCandidate{Name: name, Type: UnknownPkg})
return candidates
}

return candidates
}
139 changes: 107 additions & 32 deletions syft/pkg/apk_metadata_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"encoding/json"
"testing"

"github.com/sergi/go-diff/diffmatchpatch"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -165,168 +164,244 @@ func TestSpaceDelimitedStringSlice_UnmarshalJSON(t *testing.T) {
}
}

func TestApkMetadata_Upstream(t *testing.T) {
func TestApkMetadata_UpstreamCandidates(t *testing.T) {
tests := []struct {
name string
metadata ApkMetadata
expected string
expected []UpstreamCandidate
}{
{
name: "gocase",
metadata: ApkMetadata{
Package: "p",
},
expected: "p",
expected: []UpstreamCandidate{
{Name: "p", Type: UnknownPkg},
},
},
{
name: "same package and origin",
name: "same package and origin simple case",
metadata: ApkMetadata{
Package: "p",
OriginPackage: "p",
},
expected: "p",
expected: []UpstreamCandidate{
{Name: "p", Type: UnknownPkg},
},
},
{
name: "different package and origin",
metadata: ApkMetadata{
Package: "p",
OriginPackage: "origin",
},
expected: "origin",
expected: []UpstreamCandidate{
{Name: "origin", Type: ApkPkg},
{Name: "p", Type: UnknownPkg},
},
},
{
name: "upstream python package information as qualifier",
name: "upstream python package information as qualifier py- prefix",
metadata: ApkMetadata{
Package: "py-potatoes",
OriginPackage: "py-potatoes",
},
expected: []UpstreamCandidate{
{Name: "potatoes", Type: PythonPkg},
},
},
{
name: "upstream python package information as qualifier py3- prefix",
metadata: ApkMetadata{
Package: "py3-potatoes",
OriginPackage: "py3-potatoes",
},
expected: "potatoes",
expected: []UpstreamCandidate{
{Name: "potatoes", Type: PythonPkg},
},
},
{
name: "python package with distinct origin package",
metadata: ApkMetadata{
Package: "py3-non-existant",
OriginPackage: "abcdefg",
},
expected: "abcdefg",
expected: []UpstreamCandidate{
{Name: "abcdefg", Type: ApkPkg},
{Name: "non-existant", Type: PythonPkg},
},
},
{
name: "upstream ruby package information as qualifier",
metadata: ApkMetadata{
Package: "ruby-something",
OriginPackage: "ruby-something",
},
expected: "something",
expected: []UpstreamCandidate{
{Name: "something", Type: GemPkg},
},
},
{
name: "python package with distinct origin package",
name: "ruby package with distinct origin package",
metadata: ApkMetadata{
Package: "ruby-something",
OriginPackage: "1234567",
},
expected: "1234567",
expected: []UpstreamCandidate{
{Name: "1234567", Type: ApkPkg},
{Name: "something", Type: GemPkg},
},
},
{
name: "postgesql-15 upstream postgresql",
metadata: ApkMetadata{
Package: "postgresql-15",
},
expected: "postgresql",
expected: []UpstreamCandidate{
{Name: "postgresql", Type: UnknownPkg},
},
},
{
name: "postgesql15 upstream postgresql",
metadata: ApkMetadata{
Package: "postgresql15",
},
expected: "postgresql",
expected: []UpstreamCandidate{
{Name: "postgresql", Type: UnknownPkg},
},
},
{
name: "go-1.19 upstream go",
metadata: ApkMetadata{
Package: "go-1.19",
},
expected: "go",
expected: []UpstreamCandidate{
{Name: "go", Type: UnknownPkg},
},
},
{
name: "go1.143 upstream go",
metadata: ApkMetadata{
Package: "go1.143",
},
expected: "go",
expected: []UpstreamCandidate{
{Name: "go", Type: UnknownPkg},
},
},
{
name: "abc-101.191.23456 upstream abc",
metadata: ApkMetadata{
Package: "abc-101.191.23456",
},
expected: "abc",
expected: []UpstreamCandidate{
{Name: "abc", Type: UnknownPkg},
},
},
{
name: "abc101.191.23456 upstream abc",
metadata: ApkMetadata{
Package: "abc101.191.23456",
},
expected: "abc",
expected: []UpstreamCandidate{
{Name: "abc", Type: UnknownPkg},
},
},
{
name: "abc101-12345-1045 upstream abc101-12345",
metadata: ApkMetadata{
Package: "abc101-12345-1045",
},
expected: "abc101-12345",
expected: []UpstreamCandidate{
{Name: "abc101-12345", Type: UnknownPkg},
},
},
{
name: "abc101-a12345-1045 upstream abc101-a12345",
metadata: ApkMetadata{
Package: "abc101-a12345-1045",
},
expected: "abc101-a12345",
expected: []UpstreamCandidate{
{Name: "abc-a12345-1045", Type: UnknownPkg},
},
},
{
name: "package starting with single digit",
metadata: ApkMetadata{
Package: "3proxy",
},
expected: "3proxy",
expected: []UpstreamCandidate{
{Name: "3proxy", Type: UnknownPkg},
},
},
{
name: "package starting with multiple digits",
metadata: ApkMetadata{
Package: "356proxy",
},
expected: "356proxy",
expected: []UpstreamCandidate{
{Name: "356proxy", Type: UnknownPkg},
},
},
{
name: "package composed of only digits",
metadata: ApkMetadata{
Package: "123456",
},
expected: "123456",
expected: []UpstreamCandidate{
{Name: "123456", Type: UnknownPkg},
},
},
{
name: "ruby-3.6 upstream ruby",
metadata: ApkMetadata{
Package: "ruby-3.6",
},
expected: "ruby",
expected: []UpstreamCandidate{
{Name: "ruby", Type: UnknownPkg},
},
},
{
name: "ruby3.6 upstream ruby",
metadata: ApkMetadata{
Package: "ruby3.6",
},
expected: "ruby",
expected: []UpstreamCandidate{
{Name: "ruby", Type: UnknownPkg},
},
},
{
name: "ruby3.6-tacos upstream tacos",
metadata: ApkMetadata{
Package: "ruby3.6-tacos",
},
expected: []UpstreamCandidate{
{Name: "tacos", Type: GemPkg},
},
},
{
name: "ruby-3.6-tacos upstream tacos",
metadata: ApkMetadata{
Package: "ruby-3.6-tacos",
},
expected: []UpstreamCandidate{
{Name: "tacos", Type: GemPkg},
},
},
{
name: "abc1234jksajflksa",
metadata: ApkMetadata{
Package: "abc1234jksajflksa",
},
expected: []UpstreamCandidate{
{Name: "abc1234jksajflksa", Type: UnknownPkg},
},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual := test.metadata.Upstream()
if actual != test.expected {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(test.expected, actual, true)
t.Errorf("diff: %s", dmp.DiffPrettyText(diffs))
}
actual := test.metadata.UpstreamCandidates()
assert.Equal(t, test.expected, actual)
})
}
}
11 changes: 8 additions & 3 deletions syft/pkg/cataloger/apkdb/package.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,14 @@ func packageURL(m pkg.ApkMetadata, distro *linux.Release) string {
pkg.PURLQualifierArch: m.Architecture,
}

upstream := m.Upstream()
if upstream != "" && upstream != m.Package {
qualifiers[pkg.PURLQualifierUpstream] = upstream
upstreams := m.UpstreamCandidates()
if len(upstreams) > 0 {
// only room for one value so for now just take the first one
upstream := upstreams[0]

if upstream.Name != "" && upstream.Name != m.Package {
qualifiers[pkg.PURLQualifierUpstream] = upstream.Name
}
}

return packageurl.NewPackageURL(
Expand Down
2 changes: 1 addition & 1 deletion syft/pkg/cataloger/apkdb/package_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ func Test_PackageURL(t *testing.T) {
ID: "alpine",
VersionID: "3.4.6",
},
expected: "pkg:apk/alpine/abc101-a12345-1045@101.191.23456?arch=a&upstream=abc101-a12345&distro=alpine-3.4.6",
expected: "pkg:apk/alpine/abc101-a12345-1045@101.191.23456?arch=a&upstream=abc-a12345-1045&distro=alpine-3.4.6",
},
{
name: "wolfi distro",
Expand Down
Loading