From 087528885268ba1d9e98c6145e6e6975abdacd17 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 25 Nov 2024 12:57:30 -0500 Subject: [PATCH 1/2] add ability to map CPEs directly to packages Signed-off-by: Alex Goodman --- grype/db/v6/affected_cpe_store_test.go | 63 ++- grype/db/v6/affected_package_store.go | 192 +++++--- grype/db/v6/affected_package_store_test.go | 513 +++++++++++++++++---- grype/db/v6/models.go | 78 +++- 4 files changed, 668 insertions(+), 178 deletions(-) diff --git a/grype/db/v6/affected_cpe_store_test.go b/grype/db/v6/affected_cpe_store_test.go index a9024d295f7..47f4f365ed9 100644 --- a/grype/db/v6/affected_cpe_store_test.go +++ b/grype/db/v6/affected_cpe_store_test.go @@ -17,7 +17,7 @@ func TestAffectedCPEStore_AddAffectedCPEs(t *testing.T) { VulnerabilityID: 1, CpeID: 1, CPE: &Cpe{ - Type: "a", + Part: "a", Vendor: "vendor-1", Product: "product-1", Edition: "edition-1", @@ -80,10 +80,69 @@ func TestAffectedCPEStore_GetCPEsByProduct(t *testing.T) { } } +func TestAffectedCPEStore_PreventDuplicateCPEs(t *testing.T) { + db := setupTestStore(t).db + bw := newBlobStore(db) + s := newAffectedCPEStore(db, bw) + + cpe1 := &AffectedCPEHandle{ + VulnerabilityID: 1, + CpeID: 1, + CPE: &Cpe{ + Part: "a", + Vendor: "vendor-1", + Product: "product-1", + Edition: "edition-1", + }, + BlobValue: &AffectedPackageBlob{ + CVEs: []string{"CVE-2023-5678"}, + }, + } + + err := s.AddAffectedCPEs(cpe1) + require.NoError(t, err) + + // attempt to add a duplicate CPE with the same values + duplicateCPE := &AffectedCPEHandle{ + VulnerabilityID: 2, // different VulnerabilityID for testing + CpeID: 2, + CPE: &Cpe{ + Part: "a", // same + Vendor: "vendor-1", // same + Product: "product-1", // same + Edition: "edition-1", // same + }, + BlobValue: &AffectedPackageBlob{ + CVEs: []string{"CVE-2024-1234"}, + }, + } + + err = s.AddAffectedCPEs(duplicateCPE) + require.NoError(t, err) + + require.Equal(t, cpe1.CpeID, duplicateCPE.CpeID, "expected the CPE DB ID to be the same") + + var existingCPEs []Cpe + err = db.Find(&existingCPEs).Error + require.NoError(t, err) + require.Len(t, existingCPEs, 1, "expected only one CPE to exist") + + actualHandles, err := s.GetCPEsByProduct(cpe1.CPE.Product, &GetAffectedCPEOptions{ + PreloadCPE: true, + PreloadBlob: true, + }) + require.NoError(t, err) + expected := []AffectedCPEHandle{*cpe1, *duplicateCPE} + require.Len(t, actualHandles, len(expected), "expected both handles to be stored") + if d := cmp.Diff(expected, actualHandles); d != "" { + t.Errorf("unexpected result (-want +got):\n%s", d) + } +} + func testAffectedCPEHandle() *AffectedCPEHandle { return &AffectedCPEHandle{ CPE: &Cpe{ - Type: "application", + Part: "application", Vendor: "vendor", Product: "product", Edition: "edition", diff --git a/grype/db/v6/affected_package_store.go b/grype/db/v6/affected_package_store.go index e990bb42300..94d61aec4da 100644 --- a/grype/db/v6/affected_package_store.go +++ b/grype/db/v6/affected_package_store.go @@ -9,6 +9,7 @@ import ( "gorm.io/gorm" "github.com/anchore/grype/internal/log" + "github.com/anchore/syft/syft/cpe" ) var NoDistroSpecified = &DistroSpecifier{} @@ -18,11 +19,12 @@ var ErrDistroNotPresent = errors.New("distro not present") var ErrMultipleOSMatches = errors.New("multiple OS matches found but not allowed") type GetAffectedPackageOptions struct { - PreloadOS bool - PreloadPackage bool - PreloadBlob bool - PackageType string - Distro *DistroSpecifier + PreloadOS bool + PreloadPackage bool + PreloadPackageCPEs bool + PreloadBlob bool + PackageType string + Distro *DistroSpecifier } // DistroSpecifier is a struct that represents a distro in a way that can be used to query the affected package store. @@ -91,6 +93,7 @@ type AffectedPackageStoreWriter interface { type AffectedPackageStoreReader interface { GetAffectedPackagesByName(packageName string, config *GetAffectedPackageOptions) ([]AffectedPackageHandle, error) + GetAffectedPackagesByCPE(cpe cpe.Attributes, config *GetAffectedPackageOptions) ([]AffectedPackageHandle, error) } type affectedPackageStore struct { @@ -107,15 +110,6 @@ func newAffectedPackageStore(db *gorm.DB, bs *blobStore) *affectedPackageStore { func (s *affectedPackageStore) AddAffectedPackages(packages ...*AffectedPackageHandle) error { for _, v := range packages { - if v.Package != nil { - var existingPackage Package - result := s.db.Where("name = ? AND type = ?", v.Package.Name, v.Package.Type).FirstOrCreate(&existingPackage, v.Package) - if result.Error != nil { - return fmt.Errorf("failed to create package (name=%q type=%q): %w", v.Package.Name, v.Package.Type, result.Error) - } - v.Package = &existingPackage - } - if err := s.blobStore.addBlobable(v); err != nil { return fmt.Errorf("unable to add affected blob: %w", err) } @@ -131,28 +125,43 @@ func (s *affectedPackageStore) GetAffectedPackagesByName(packageName string, con config = &GetAffectedPackageOptions{} } - log.WithFields("name", packageName, "distro", distroDisplay(config.Distro)).Trace("fetching AffectedPackage record") + log.WithFields("name", packageName, "distro", distroDisplay(config.Distro)).Trace("fetching AffectedPackage by name record") - if hasDistroSpecified(config.Distro) { - return s.getPackageByNameAndDistro(packageName, *config) + return s.getAffectedPackagesWithOptions( + s.handlePackageName(s.db, packageName), + config, + ) +} + +func (s *affectedPackageStore) GetAffectedPackagesByCPE(cpe cpe.Attributes, config *GetAffectedPackageOptions) ([]AffectedPackageHandle, error) { + if config == nil { + config = &GetAffectedPackageOptions{} } - return s.getNonDistroPackageByName(packageName, *config) -} + log.WithFields("cpe", cpe.String(), "distro", distroDisplay(config.Distro)).Trace("fetching AffectedPackage by CPE record") -func (s *affectedPackageStore) getNonDistroPackageByName(packageName string, config GetAffectedPackageOptions) ([]AffectedPackageHandle, error) { - var pkgs []AffectedPackageHandle - query := s.db.Joins("JOIN packages ON affected_package_handles.package_id = packages.id") + return s.getAffectedPackagesWithOptions( + s.handlePackageCPE(s.db, cpe), + config) +} - if config.Distro != AnyDistroSpecified { - query = query.Where("operating_system_id IS NULL") +func (s *affectedPackageStore) getAffectedPackagesWithOptions(query *gorm.DB, config *GetAffectedPackageOptions) ([]AffectedPackageHandle, error) { + if config == nil { + config = &GetAffectedPackageOptions{} } - query = s.handlePackage(query, packageName, config) - query = s.handlePreload(query, config) + query = s.handlePackageOptions(query, *config) - err := query.Find(&pkgs).Error + var err error + query, err = s.handleDistroOptions(query, *config) + if err != nil { + return nil, err + } + query = s.handlePreload(query, *config) + + var pkgs []AffectedPackageHandle + err = query.Find(&pkgs).Error if err != nil { return nil, fmt.Errorf("unable to fetch non-distro affected package record: %w", err) } @@ -169,10 +178,74 @@ func (s *affectedPackageStore) getNonDistroPackageByName(packageName string, con return pkgs, nil } -func (s *affectedPackageStore) getPackageByNameAndDistro(packageName string, config GetAffectedPackageOptions) ([]AffectedPackageHandle, error) { +func (s *affectedPackageStore) handlePackageName(query *gorm.DB, packageName string) *gorm.DB { + return query.Joins("JOIN packages ON affected_package_handles.package_id = packages.id").Where("packages.name = ?", packageName) +} + +func (s *affectedPackageStore) handlePackageCPE(query *gorm.DB, c cpe.Attributes) *gorm.DB { + query = query.Joins("JOIN packages ON affected_package_handles.package_id = packages.id").Joins("JOIN cpes ON packages.id = cpes.package_id") + + if c.Part != cpe.Any { + query = query.Where("cpes.part = ?", c.Part) + } + + if c.Vendor != cpe.Any { + query = query.Where("cpes.vendor = ?", c.Vendor) + } + + if c.Product != cpe.Any { + query = query.Where("cpes.product = ?", c.Product) + } + + if c.Version != cpe.Any { + query = query.Where("cpes.version = ?", c.Version) + } + + if c.Update != cpe.Any { + query = query.Where("cpes.update = ?", c.Update) + } + + if c.Edition != cpe.Any { + query = query.Where("cpes.edition = ?", c.Edition) + } + + if c.Language != cpe.Any { + query = query.Where("cpes.language = ?", c.Language) + } + + if c.SWEdition != cpe.Any { + query = query.Where("cpes.sw_edition = ?", c.SWEdition) + } + + if c.TargetSW != cpe.Any { + query = query.Where("cpes.target_sw = ?", c.TargetSW) + } + + if c.TargetHW != cpe.Any { + query = query.Where("cpes.target_hw = ?", c.TargetHW) + } + + if c.Other != cpe.Any { + query = query.Where("cpes.other = ?", c.Other) + } + + return query +} + +func (s *affectedPackageStore) handlePackageOptions(query *gorm.DB, config GetAffectedPackageOptions) *gorm.DB { + if config.PackageType != "" { + query = query.Where("packages.type = ?", config.PackageType) + } + + return query +} + +func (s *affectedPackageStore) handleDistroOptions(query *gorm.DB, config GetAffectedPackageOptions) (*gorm.DB, error) { var resolvedDistros []OperatingSystem var err error - if config.Distro != NoDistroSpecified || config.Distro != AnyDistroSpecified { + + switch { + case hasDistroSpecified(config.Distro): resolvedDistros, err = s.resolveDistro(*config.Distro) if err != nil { return nil, fmt.Errorf("unable to resolve distro: %w", err) @@ -184,31 +257,28 @@ func (s *affectedPackageStore) getPackageByNameAndDistro(packageName string, con case len(resolvedDistros) > 1 && !config.Distro.AllowMultiple: return nil, ErrMultipleOSMatches } + case config.Distro == AnyDistroSpecified: + // TODO: one enhancement we may want to do later is "has OS defined but is not specific" which this does NOT cover. This is "may or may not have an OS defined" which is different. + return query, nil + case *config.Distro == *NoDistroSpecified: + return query.Where("operating_system_id IS NULL"), nil } - var pkgs []AffectedPackageHandle - query := s.db.Joins("JOIN packages ON affected_package_handles.package_id = packages.id"). - Joins("JOIN operating_systems ON affected_package_handles.operating_system_id = operating_systems.id") + query = query.Joins("JOIN operating_systems ON affected_package_handles.operating_system_id = operating_systems.id") - query = s.handlePackage(query, packageName, config) - query = s.handleDistros(query, resolvedDistros) - query = s.handlePreload(query, config) - - err = query.Find(&pkgs).Error - if err != nil { - return nil, fmt.Errorf("unable to fetch affected package record: %w", err) - } - - if config.PreloadBlob { - for i := range pkgs { - err := s.attachBlob(&pkgs[i]) - if err != nil { - return nil, fmt.Errorf("unable to attach blob %#v: %w", pkgs[i], err) + var count int + for _, o := range resolvedDistros { + if o.ID != 0 { + if count == 0 { + query = query.Where("operating_systems.id = ?", o.ID) + } else { + query = query.Or("operating_systems.id = ?", o.ID) } + count++ } } - return pkgs, nil + return query, nil } func (s *affectedPackageStore) resolveDistro(d DistroSpecifier) ([]OperatingSystem, error) { @@ -355,33 +425,13 @@ func (s *affectedPackageStore) applyAlias(d *DistroSpecifier) error { return nil } -func (s *affectedPackageStore) handlePackage(query *gorm.DB, packageName string, config GetAffectedPackageOptions) *gorm.DB { - query = query.Where("packages.name = ?", packageName) - - if config.PackageType != "" { - query = query.Where("packages.type = ?", config.PackageType) - } - return query -} - -func (s *affectedPackageStore) handleDistros(query *gorm.DB, resolvedDistros []OperatingSystem) *gorm.DB { - var count int - for _, o := range resolvedDistros { - if o.ID != 0 { - if count == 0 { - query = query.Where("operating_systems.id = ?", o.ID) - } else { - query = query.Or("operating_systems.id = ?", o.ID) - } - count++ - } - } - return query -} - func (s *affectedPackageStore) handlePreload(query *gorm.DB, config GetAffectedPackageOptions) *gorm.DB { if config.PreloadPackage { query = query.Preload("Package") + + if config.PreloadPackageCPEs { + query = query.Preload("Package.CPEs") + } } if config.PreloadOS { diff --git a/grype/db/v6/affected_package_store_test.go b/grype/db/v6/affected_package_store_test.go index a73ecc34259..c17858e29cb 100644 --- a/grype/db/v6/affected_package_store_test.go +++ b/grype/db/v6/affected_package_store_test.go @@ -8,38 +8,431 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cpe" ) +type affectedPackageHandlePreloadConfig struct { + name string + PreloadOS bool + PreloadPackage bool + PreloadBlob bool + prepExpectations func(*testing.T, []AffectedPackageHandle) []AffectedPackageHandle +} + +func defaultAffectedPackageHandlePreloadCases() []affectedPackageHandlePreloadConfig { + return []affectedPackageHandlePreloadConfig{ + { + name: "preload-all", + PreloadOS: true, + PreloadPackage: true, + PreloadBlob: true, + prepExpectations: func(t *testing.T, in []AffectedPackageHandle) []AffectedPackageHandle { + for _, a := range in { + if a.OperatingSystemID != nil { + require.NotNil(t, a.OperatingSystem) + } + require.NotNil(t, a.Package) + require.NotNil(t, a.BlobValue) + } + return in + }, + }, + { + name: "preload-none", + prepExpectations: func(t *testing.T, in []AffectedPackageHandle) []AffectedPackageHandle { + var out []AffectedPackageHandle + for _, v := range in { + if v.OperatingSystem == nil && v.BlobValue == nil && v.Package == nil { + t.Skip("preload already matches expectation") + } + v.OperatingSystem = nil + v.Package = nil + v.BlobValue = nil + out = append(out, v) + } + return out + }, + }, + { + name: "preload-os-only", + PreloadOS: true, + prepExpectations: func(t *testing.T, in []AffectedPackageHandle) []AffectedPackageHandle { + var out []AffectedPackageHandle + for _, a := range in { + if a.OperatingSystemID != nil { + require.NotNil(t, a.OperatingSystem) + } + if a.Package == nil && a.BlobValue == nil { + t.Skip("preload already matches expectation") + } + a.Package = nil + a.BlobValue = nil + out = append(out, a) + } + return out + }, + }, + { + name: "preload-package-only", + PreloadPackage: true, + prepExpectations: func(t *testing.T, in []AffectedPackageHandle) []AffectedPackageHandle { + var out []AffectedPackageHandle + for _, a := range in { + require.NotNil(t, a.Package) + if a.OperatingSystem == nil && a.BlobValue == nil { + t.Skip("preload already matches expectation") + } + a.OperatingSystem = nil + a.BlobValue = nil + out = append(out, a) + } + return out + }, + }, + { + name: "preload-blob-only", + PreloadBlob: true, + prepExpectations: func(t *testing.T, in []AffectedPackageHandle) []AffectedPackageHandle { + var out []AffectedPackageHandle + for _, a := range in { + if a.OperatingSystem == nil && a.Package == nil { + t.Skip("preload already matches expectation") + } + a.OperatingSystem = nil + a.Package = nil + out = append(out, a) + } + return out + }, + }, + } +} + func TestAffectedPackageStore_AddAffectedPackages(t *testing.T) { + setupAffectedPackageStore := func(t *testing.T) *affectedPackageStore { + db := setupTestStore(t).db + return newAffectedPackageStore(db, newBlobStore(db)) + } + + setupTestStoreWithPackages := func(t *testing.T) (*AffectedPackageHandle, *AffectedPackageHandle, *affectedPackageStore) { + pkg1 := &AffectedPackageHandle{ + Package: &Package{Name: "pkg1", Type: "type1"}, + BlobValue: &AffectedPackageBlob{ + CVEs: []string{"CVE-2023-1234"}, + }, + } + + pkg2 := testDistro1AffectedPackage2Handle() + + return pkg1, pkg2, setupAffectedPackageStore(t) + } + + t.Run("no preloading", func(t *testing.T) { + pkg1, pkg2, s := setupTestStoreWithPackages(t) + + err := s.AddAffectedPackages(pkg1, pkg2) + require.NoError(t, err) + + var result1 AffectedPackageHandle + err = s.db.Where("package_id = ?", pkg1.PackageID).First(&result1).Error + require.NoError(t, err) + assert.Equal(t, pkg1.PackageID, result1.PackageID) + assert.Equal(t, pkg1.BlobID, result1.BlobID) + require.Nil(t, result1.BlobValue) // no preloading on fetch + + var result2 AffectedPackageHandle + err = s.db.Where("package_id = ?", pkg2.PackageID).First(&result2).Error + require.NoError(t, err) + assert.Equal(t, pkg2.PackageID, result2.PackageID) + assert.Equal(t, pkg2.BlobID, result2.BlobID) + require.Nil(t, result2.BlobValue) + }) + + t.Run("preloading", func(t *testing.T) { + pkg1, pkg2, s := setupTestStoreWithPackages(t) + + err := s.AddAffectedPackages(pkg1, pkg2) + require.NoError(t, err) + + options := &GetAffectedPackageOptions{ + PreloadOS: true, + PreloadPackage: true, + PreloadBlob: true, + } + + results, err := s.GetAffectedPackagesByName(pkg1.Package.Name, options) + require.NoError(t, err) + require.Len(t, results, 1) + + result := results[0] + require.NotNil(t, result.Package) + require.NotNil(t, result.BlobValue) + assert.Nil(t, result.OperatingSystem) // pkg1 has no OS + }) + + t.Run("preload CPEs", func(t *testing.T) { + pkg1, _, s := setupTestStoreWithPackages(t) + + cpe := Cpe{ + Part: "a", + Vendor: "vendor1", + Product: "product1", + } + pkg1.Package.CPEs = []Cpe{cpe} + + err := s.AddAffectedPackages(pkg1) + require.NoError(t, err) + + options := &GetAffectedPackageOptions{ + PreloadPackage: true, + PreloadPackageCPEs: true, + } + + results, err := s.GetAffectedPackagesByName(pkg1.Package.Name, options) + require.NoError(t, err) + require.Len(t, results, 1) + + result := results[0] + require.NotNil(t, result.Package) + + // the IDs should have been set, and there is only one, so we know the correct values + cpe.ID = 1 + cpe.PackageID = idRef(1) + + if d := cmp.Diff([]Cpe{cpe}, result.Package.CPEs); d != "" { + t.Errorf("unexpected result (-want +got):\n%s", d) + } + }) + + t.Run("Package deduplication", func(t *testing.T) { + pkg1 := &AffectedPackageHandle{ + Package: &Package{Name: "pkg1", Type: "type1"}, + BlobValue: &AffectedPackageBlob{ + CVEs: []string{"CVE-2023-1234"}, + }, + } + + pkg2 := &AffectedPackageHandle{ + Package: &Package{Name: "pkg1", Type: "type1"}, // same! + BlobValue: &AffectedPackageBlob{ + CVEs: []string{"CVE-2023-56789"}, + }, + } + + s := setupAffectedPackageStore(t) + err := s.AddAffectedPackages(pkg1, pkg2) + require.NoError(t, err) + + var pkgs []Package + err = s.db.Find(&pkgs).Error + require.NoError(t, err) + + expected := []Package{ + *pkg1.Package, + } + + if d := cmp.Diff(expected, pkgs); d != "" { + t.Errorf("unexpected result (-want +got):\n%s", d) + } + }) + + t.Run("same package with multiple CPEs", func(t *testing.T) { + cpe1 := Cpe{ + Part: "a", + Vendor: "vendor1", + Product: "product1", + } + + cpe2 := Cpe{ + Part: "a", + Vendor: "vendor2", + Product: "product2", + } + + pkg1 := &AffectedPackageHandle{ + Package: &Package{Name: "pkg1", Type: "type1", CPEs: []Cpe{cpe1}}, + BlobValue: &AffectedPackageBlob{ + CVEs: []string{"CVE-2023-1234"}, + }, + } + + pkg2 := &AffectedPackageHandle{ + Package: &Package{Name: "pkg1", Type: "type1", CPEs: []Cpe{cpe1, cpe2}}, // duplicate CPE + additional CPE + BlobValue: &AffectedPackageBlob{ + CVEs: []string{"CVE-2023-56789"}, + }, + } + + s := setupAffectedPackageStore(t) + err := s.AddAffectedPackages(pkg1, pkg2) + require.NoError(t, err) + + var pkgs []Package + err = s.db.Preload("CPEs").Find(&pkgs).Error + require.NoError(t, err) + + expPkg := *pkg1.Package + expPkg.ID = 1 + cpe1.ID = 1 + cpe1.PackageID = idRef(1) + cpe2.ID = 2 + cpe2.PackageID = idRef(1) + expPkg.CPEs = []Cpe{cpe1, cpe2} + + expected := []Package{ + expPkg, + } + + if d := cmp.Diff(expected, pkgs); d != "" { + t.Errorf("unexpected result (-want +got):\n%s", d) + } + + expectedCPEs := []Cpe{cpe1, cpe2} + var cpeResults []Cpe + err = s.db.Find(&cpeResults).Error + require.NoError(t, err) + if d := cmp.Diff(expectedCPEs, cpeResults); d != "" { + t.Errorf("unexpected result (-want +got):\n%s", d) + } + + }) + + t.Run("dont allow same CPE to belong to multiple packages", func(t *testing.T) { + cpe1 := Cpe{ + Part: "a", + Vendor: "vendor1", + Product: "product1", + } + + cpe2 := Cpe{ + Part: "a", + Vendor: "vendor2", + Product: "product2", + } + + pkg1 := &AffectedPackageHandle{ + Package: &Package{Name: "pkg1", Type: "type1", CPEs: []Cpe{cpe1}}, + BlobValue: &AffectedPackageBlob{ + CVEs: []string{"CVE-2023-1234"}, + }, + } + + pkg2 := &AffectedPackageHandle{ + Package: &Package{Name: "pkg2", Type: "type1", CPEs: []Cpe{cpe1, cpe2}}, // overlapping CPEs for different packages + BlobValue: &AffectedPackageBlob{ + CVEs: []string{"CVE-2023-56789"}, + }, + } + + s := setupAffectedPackageStore(t) + err := s.AddAffectedPackages(pkg1, pkg2) + require.ErrorContains(t, err, "CPE already exists for a different package") + }) +} + +func TestAffectedPackageStore_GetAffectedPackagesByCPE(t *testing.T) { db := setupTestStore(t).db bs := newBlobStore(db) s := newAffectedPackageStore(db, bs) + cpe1 := Cpe{Part: "a", Vendor: "vendor1", Product: "product1"} + cpe2 := Cpe{Part: "a", Vendor: "vendor2", Product: "product2"} pkg1 := &AffectedPackageHandle{ - Package: &Package{Name: "pkg1", Type: "type1"}, + Package: &Package{Name: "pkg1", Type: "type1", CPEs: []Cpe{cpe1}}, BlobValue: &AffectedPackageBlob{ CVEs: []string{"CVE-2023-1234"}, }, } - - pkg2 := testDistro1AffectedPackage2Handle() + pkg2 := &AffectedPackageHandle{ + Package: &Package{Name: "pkg2", Type: "type2", CPEs: []Cpe{cpe2}}, + BlobValue: &AffectedPackageBlob{ + CVEs: []string{"CVE-2023-5678"}, + }, + } err := s.AddAffectedPackages(pkg1, pkg2) require.NoError(t, err) - var result1 AffectedPackageHandle - err = db.Where("package_id = ?", pkg1.PackageID).First(&result1).Error - require.NoError(t, err) - assert.Equal(t, pkg1.PackageID, result1.PackageID) - assert.Equal(t, pkg1.BlobID, result1.BlobID) - require.Nil(t, result1.BlobValue) // no preloading on fetch + tests := []struct { + name string + cpe cpe.Attributes + options *GetAffectedPackageOptions + expected []AffectedPackageHandle + wantErr require.ErrorAssertionFunc + }{ + { + name: "full match CPE", + cpe: cpe.Attributes{ + Part: "a", + Vendor: "vendor1", + Product: "product1", + }, + options: &GetAffectedPackageOptions{ + PreloadPackageCPEs: true, + PreloadPackage: true, + PreloadBlob: true, + }, + expected: []AffectedPackageHandle{*pkg1}, + }, + { + name: "partial match CPE", + cpe: cpe.Attributes{ + Part: "a", + Vendor: "vendor2", + }, + options: &GetAffectedPackageOptions{ + PreloadPackageCPEs: true, + PreloadPackage: true, + PreloadBlob: true, + }, + expected: []AffectedPackageHandle{*pkg2}, + }, + { + name: "missing attributes", + cpe: cpe.Attributes{ + Part: "a", + }, + options: &GetAffectedPackageOptions{ + PreloadPackageCPEs: true, + PreloadPackage: true, + PreloadBlob: true, + }, + expected: []AffectedPackageHandle{*pkg1, *pkg2}, + }, + { + name: "no matches", + cpe: cpe.Attributes{ + Part: "a", + Vendor: "unknown_vendor", + Product: "unknown_product", + }, + options: &GetAffectedPackageOptions{ + PreloadPackageCPEs: true, + PreloadPackage: true, + PreloadBlob: true, + }, + expected: nil, + }, + } - var result2 AffectedPackageHandle - err = db.Where("package_id = ?", pkg2.PackageID).First(&result2).Error - require.NoError(t, err) - assert.Equal(t, pkg2.PackageID, result2.PackageID) - assert.Equal(t, pkg2.BlobID, result2.BlobID) - assert.Nil(t, result2.BlobValue) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + + result, err := s.GetAffectedPackagesByCPE(tt.cpe, tt.options) + tt.wantErr(t, err) + if err != nil { + return + } + if d := cmp.Diff(tt.expected, result, cmpopts.EquateEmpty()); d != "" { + t.Errorf(fmt.Sprintf("unexpected result: %s", d)) + } + + }) + } } func TestAffectedPackageStore_GetAffectedPackagesByName(t *testing.T) { @@ -133,95 +526,12 @@ func TestAffectedPackageStore_GetAffectedPackagesByName(t *testing.T) { }, } - type preloadConfig struct { - name string - PreloadOS bool - PreloadPackage bool - PreloadBlob bool - prepExpectations func(*testing.T, []AffectedPackageHandle) []AffectedPackageHandle - } - - preloadCases := []preloadConfig{ - { - name: "preload-all", - PreloadOS: true, - PreloadPackage: true, - PreloadBlob: true, - prepExpectations: func(t *testing.T, in []AffectedPackageHandle) []AffectedPackageHandle { - for _, a := range in { - if a.OperatingSystemID != nil { - require.NotNil(t, a.OperatingSystem) - } - require.NotNil(t, a.Package) - require.NotNil(t, a.BlobValue) - } - return in - }, - }, - { - name: "preload-none", - prepExpectations: func(t *testing.T, in []AffectedPackageHandle) []AffectedPackageHandle { - var out []AffectedPackageHandle - for _, v := range in { - v.OperatingSystem = nil - v.Package = nil - v.BlobValue = nil - out = append(out, v) - } - return out - }, - }, - { - name: "preload-os-only", - PreloadOS: true, - prepExpectations: func(t *testing.T, in []AffectedPackageHandle) []AffectedPackageHandle { - var out []AffectedPackageHandle - for _, a := range in { - if a.OperatingSystemID != nil { - require.NotNil(t, a.OperatingSystem) - } - a.Package = nil - a.BlobValue = nil - out = append(out, a) - } - return out - }, - }, - { - name: "preload-package-only", - PreloadPackage: true, - prepExpectations: func(t *testing.T, in []AffectedPackageHandle) []AffectedPackageHandle { - var out []AffectedPackageHandle - for _, a := range in { - require.NotNil(t, a.Package) - a.OperatingSystem = nil - a.BlobValue = nil - out = append(out, a) - } - return out - }, - }, - { - name: "preload-blob-only", - PreloadBlob: true, - prepExpectations: func(t *testing.T, in []AffectedPackageHandle) []AffectedPackageHandle { - var out []AffectedPackageHandle - for _, a := range in { - a.OperatingSystem = nil - a.Package = nil - out = append(out, a) - } - return out - }, - }, - } - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if tt.wantErr == nil { tt.wantErr = require.NoError } - for _, pc := range preloadCases { + for _, pc := range defaultAffectedPackageHandlePreloadCases() { t.Run(pc.name, func(t *testing.T) { opts := tt.options @@ -593,3 +903,8 @@ func expectErrIs(t *testing.T, expected error) require.ErrorAssertionFunc { func strRef(s string) *string { return &s } + +func idRef(i int64) *ID { + v := ID(i) + return &v +} diff --git a/grype/db/v6/models.go b/grype/db/v6/models.go index 7d4a74a287b..d60bb897d17 100644 --- a/grype/db/v6/models.go +++ b/grype/db/v6/models.go @@ -115,7 +115,11 @@ func (v *VulnerabilityHandle) setBlobID(id ID) { // package related search tables ////////////////////////////////////////////////////// -// AffectedPackageHandle represents a single package affected by the specified vulnerability. +// AffectedPackageHandle represents a single package affected by the specified vulnerability. A package here is a +// name within a known ecosystem, such as "python" or "golang". It is important to note that this table relates +// vulnerabilities to resolved packages. There are cases when we have package identifiers but are not resolved to +// packages; for example, when we have a CPE but not a clear understanding of the package ecosystem and authoritative +// name (which might or might not be the product name in the CPE), in which case AffectedCPEHandle should be used. type AffectedPackageHandle struct { ID ID `gorm:"column:id;primaryKey"` VulnerabilityID ID `gorm:"column:vulnerability_id;not null"` @@ -143,6 +147,53 @@ type Package struct { ID ID `gorm:"column:id;primaryKey"` Type string `gorm:"column:type;index:idx_package,unique"` Name string `gorm:"column:name;index:idx_package,unique"` + + CPEs []Cpe `gorm:"foreignKey:PackageID;constraint:OnDelete:CASCADE;"` +} + +func (p *Package) BeforeCreate(tx *gorm.DB) (err error) { + var existingPackage Package + result := tx.Where("type = ? AND name = ?", p.Type, p.Name).First(&existingPackage) + if result.Error == nil { + // package exists; merge CPEs + for i, newCPE := range p.CPEs { + // if the CPE already exists, then we should use the existing record + var existingCPE Cpe + cpeResult := cpeWhereClause(tx, &newCPE).First(&existingCPE) + if cpeResult.Error == nil { + // if the record already exists, then we should use the existing record + newCPE = existingCPE + p.CPEs[i] = newCPE + + if existingCPE.PackageID == nil { + log.WithFields("cpe", existingCPE, "pkg", existingPackage).Warn("CPE exists but was not associated with an already existing package until now") + continue + } + + if *existingCPE.PackageID != existingPackage.ID { + return fmt.Errorf("CPE already exists for a different package (pkg=%q, existing_pkg=%q): %q", p, existingPackage, newCPE) + } + continue + } + + // if the CPE does not exist, proceed with creating it + newCPE.PackageID = &existingPackage.ID + p.CPEs[i] = newCPE + + if err := tx.Create(&newCPE).Error; err != nil { + return fmt.Errorf("failed to create CPE %q for package %q: %w", newCPE, existingPackage, err) + } + } + // use the existing package instead of creating a new one + *p = existingPackage + return nil + } + + // if the package does not exist, proceed with creating it + for i := range p.CPEs { + p.CPEs[i].PackageID = &p.ID + } + return nil } type OperatingSystem struct { @@ -194,7 +245,10 @@ func (os *OperatingSystemAlias) BeforeCreate(_ *gorm.DB) (err error) { // CPE related search tables ////////////////////////////////////////////////////// -// AffectedCPEHandle represents a single CPE affected by the specified vulnerability +// AffectedCPEHandle represents a single CPE affected by the specified vulnerability. Note the CPEs in this table +// must NOT be resolvable to Packages (use AffectedPackageHandle for that). This table is used when the CPE is known, +// but we do not have a clear understanding of the package ecosystem or authoritative name, so we can still +// find vulnerabilities by these identifiers but not assert they are related to an entry in the Packages table. type AffectedCPEHandle struct { ID ID `gorm:"column:id;primaryKey"` VulnerabilityID ID `gorm:"column:vulnerability_id;not null"` @@ -217,9 +271,10 @@ func (v *AffectedCPEHandle) setBlobID(id ID) { type Cpe struct { // TODO: what about different CPE versions? - ID ID `gorm:"primaryKey"` + ID ID `gorm:"primaryKey"` + PackageID *ID `gorm:"column:package_id;index"` - Type string `gorm:"column:type;not null;index:idx_cpe,unique"` + Part string `gorm:"column:part;not null;index:idx_cpe,unique"` Vendor string `gorm:"column:vendor;index:idx_cpe,unique"` Product string `gorm:"column:product;not null;index:idx_cpe,unique"` Edition string `gorm:"column:edition;index:idx_cpe,unique"` @@ -231,16 +286,27 @@ type Cpe struct { } func (c Cpe) String() string { - return fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%s", c.Type, c.Vendor, c.Product, c.Edition, c.Language, c.SoftwareEdition, c.TargetHardware, c.TargetSoftware, c.Other) + return fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%s", c.Part, c.Vendor, c.Product, c.Edition, c.Language, c.SoftwareEdition, c.TargetHardware, c.TargetSoftware, c.Other) } func (c *Cpe) BeforeCreate(tx *gorm.DB) (err error) { // if the name, major version, and minor version already exist in the table then we should not insert a new record var existing Cpe - result := tx.Where("type = ? AND vendor = ? AND product = ? AND edition = ? AND language = ? AND software_edition = ? AND target_hardware = ? AND target_software = ? AND other = ?", c.Type, c.Vendor, c.Product, c.Edition, c.Language, c.SoftwareEdition, c.TargetHardware, c.TargetSoftware, c.Other).First(&existing) + result := cpeWhereClause(tx, c).First(&existing) if result.Error == nil { + if c.PackageID != nil && c.PackageID != existing.PackageID { + return fmt.Errorf("CPE already exists for a different package (pkg=%d, existing_pkg=%d): %q", c.PackageID, existing.PackageID, c) + } + // if the record already exists, then we should use the existing record *c = existing } return nil } + +func cpeWhereClause(tx *gorm.DB, c *Cpe) *gorm.DB { + if c == nil { + return tx + } + return tx.Where("part = ? AND vendor = ? AND product = ? AND edition = ? AND language = ? AND software_edition = ? AND target_hardware = ? AND target_software = ? AND other = ?", c.Part, c.Vendor, c.Product, c.Edition, c.Language, c.SoftwareEdition, c.TargetHardware, c.TargetSoftware, c.Other) +} From 56ab5f54ae50fc8381110ec56a131375f5d54150 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Wed, 27 Nov 2024 16:55:41 -0500 Subject: [PATCH 2/2] rm search by cpe version info Signed-off-by: Alex Goodman --- grype/db/v6/affected_package_store.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/grype/db/v6/affected_package_store.go b/grype/db/v6/affected_package_store.go index 94d61aec4da..ea0fd0dafe4 100644 --- a/grype/db/v6/affected_package_store.go +++ b/grype/db/v6/affected_package_store.go @@ -197,14 +197,6 @@ func (s *affectedPackageStore) handlePackageCPE(query *gorm.DB, c cpe.Attributes query = query.Where("cpes.product = ?", c.Product) } - if c.Version != cpe.Any { - query = query.Where("cpes.version = ?", c.Version) - } - - if c.Update != cpe.Any { - query = query.Where("cpes.update = ?", c.Update) - } - if c.Edition != cpe.Any { query = query.Where("cpes.edition = ?", c.Edition) }