From 270b85161574358688693ef2c66b93b627de9634 Mon Sep 17 00:00:00 2001 From: Tatiana Bradley Date: Fri, 26 Jan 2024 16:53:22 -0500 Subject: [PATCH] all: remove all code related to legacy vulndb format This change removes all code related to generating and validating vulnerability databases in the legacy format. Fixes golang/go#65320 Change-Id: I83522efec90aa04df54976d5c29d485acd13e53c Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/558658 Auto-Submit: Tatiana Bradley LUCI-TryBot-Result: Go LUCI Reviewed-by: Damien Neil --- cmd/checkdb/main.go | 28 +- cmd/checkdeploy/main.go | 18 +- cmd/dbdiff/main.go | 25 -- cmd/gendb/main.go | 26 +- internal/database/legacydb/database.go | 159 -------- internal/database/legacydb/database_test.go | 96 ----- internal/database/legacydb/diff.go | 30 -- internal/database/legacydb/equivalent.go | 65 ---- internal/database/legacydb/equivalent_test.go | 69 ---- internal/database/legacydb/generate.go | 31 -- internal/database/legacydb/generate_test.go | 94 ----- internal/database/legacydb/load.go | 264 -------------- internal/database/legacydb/load_test.go | 343 ------------------ .../testdata/db/existing/ID/GO-1999-0001.json | 57 --- .../testdata/db/existing/ID/GO-2000-0002.json | 51 --- .../testdata/db/existing/ID/index.json | 4 - .../testdata/db/existing/aliases.json | 8 - .../db/existing/example.com/module.json | 59 --- .../db/existing/example.com/module2.json | 53 --- .../legacydb/testdata/db/existing/index.json | 4 - .../db/missing-file/ID/GO-1999-0001.json | 57 --- .../db/missing-file/ID/GO-2000-0002.json | 51 --- .../testdata/db/missing-file/ID/index.json | 4 - .../testdata/db/missing-file/aliases.json | 8 - .../db/missing-file/example.com/module.json | 59 --- .../testdata/db/missing-file/index.json | 4 - .../db/unexpected-file/ID/GO-1999-0001.json | 57 --- .../db/unexpected-file/ID/GO-2000-0002.json | 51 --- .../testdata/db/unexpected-file/ID/index.json | 3 - .../testdata/db/unexpected-file/aliases.json | 5 - .../unexpected-file/example.com/module.json | 59 --- .../testdata/db/unexpected-file/index.json | 3 - .../testdata/db/valid/ID/GO-1999-0001.json | 57 --- .../testdata/db/valid/ID/GO-2000-0002.json | 51 --- .../testdata/db/valid/ID/GO-2000-0003.json | 52 --- .../legacydb/testdata/db/valid/ID/index.json | 5 - .../legacydb/testdata/db/valid/aliases.json | 14 - .../testdata/db/valid/example.com/module.json | 59 --- .../db/valid/example.com/module2.json | 105 ------ .../legacydb/testdata/db/valid/index.json | 4 - .../database/legacydb/testdata/repo.txtar | 167 --------- internal/database/legacydb/validate.go | 48 --- internal/database/legacydb/validate_test.go | 135 ------- internal/database/legacydb/write.go | 118 ------ internal/database/legacydb/write_test.go | 59 --- 45 files changed, 14 insertions(+), 2705 deletions(-) delete mode 100644 cmd/dbdiff/main.go delete mode 100644 internal/database/legacydb/database.go delete mode 100644 internal/database/legacydb/database_test.go delete mode 100644 internal/database/legacydb/diff.go delete mode 100644 internal/database/legacydb/equivalent.go delete mode 100644 internal/database/legacydb/equivalent_test.go delete mode 100644 internal/database/legacydb/generate.go delete mode 100644 internal/database/legacydb/generate_test.go delete mode 100644 internal/database/legacydb/load.go delete mode 100644 internal/database/legacydb/load_test.go delete mode 100644 internal/database/legacydb/testdata/db/existing/ID/GO-1999-0001.json delete mode 100644 internal/database/legacydb/testdata/db/existing/ID/GO-2000-0002.json delete mode 100644 internal/database/legacydb/testdata/db/existing/ID/index.json delete mode 100644 internal/database/legacydb/testdata/db/existing/aliases.json delete mode 100644 internal/database/legacydb/testdata/db/existing/example.com/module.json delete mode 100644 internal/database/legacydb/testdata/db/existing/example.com/module2.json delete mode 100644 internal/database/legacydb/testdata/db/existing/index.json delete mode 100644 internal/database/legacydb/testdata/db/missing-file/ID/GO-1999-0001.json delete mode 100644 internal/database/legacydb/testdata/db/missing-file/ID/GO-2000-0002.json delete mode 100644 internal/database/legacydb/testdata/db/missing-file/ID/index.json delete mode 100644 internal/database/legacydb/testdata/db/missing-file/aliases.json delete mode 100644 internal/database/legacydb/testdata/db/missing-file/example.com/module.json delete mode 100644 internal/database/legacydb/testdata/db/missing-file/index.json delete mode 100644 internal/database/legacydb/testdata/db/unexpected-file/ID/GO-1999-0001.json delete mode 100644 internal/database/legacydb/testdata/db/unexpected-file/ID/GO-2000-0002.json delete mode 100644 internal/database/legacydb/testdata/db/unexpected-file/ID/index.json delete mode 100644 internal/database/legacydb/testdata/db/unexpected-file/aliases.json delete mode 100644 internal/database/legacydb/testdata/db/unexpected-file/example.com/module.json delete mode 100644 internal/database/legacydb/testdata/db/unexpected-file/index.json delete mode 100644 internal/database/legacydb/testdata/db/valid/ID/GO-1999-0001.json delete mode 100644 internal/database/legacydb/testdata/db/valid/ID/GO-2000-0002.json delete mode 100644 internal/database/legacydb/testdata/db/valid/ID/GO-2000-0003.json delete mode 100644 internal/database/legacydb/testdata/db/valid/ID/index.json delete mode 100644 internal/database/legacydb/testdata/db/valid/aliases.json delete mode 100644 internal/database/legacydb/testdata/db/valid/example.com/module.json delete mode 100644 internal/database/legacydb/testdata/db/valid/example.com/module2.json delete mode 100644 internal/database/legacydb/testdata/db/valid/index.json delete mode 100644 internal/database/legacydb/testdata/repo.txtar delete mode 100644 internal/database/legacydb/validate.go delete mode 100644 internal/database/legacydb/validate_test.go delete mode 100644 internal/database/legacydb/write.go delete mode 100644 internal/database/legacydb/write_test.go diff --git a/cmd/checkdb/main.go b/cmd/checkdb/main.go index 1a4fe5e44..e4acdc34e 100644 --- a/cmd/checkdb/main.go +++ b/cmd/checkdb/main.go @@ -11,12 +11,6 @@ import ( "log" db "golang.org/x/vulndb/internal/database" - "golang.org/x/vulndb/internal/database/legacydb" -) - -var ( - v1 = flag.Bool("v1", true, "if true, check with respect to v1 database schema") - legacy = flag.Bool("legacy", false, "if true, check with respect to legacy database schema") ) func main() { @@ -25,24 +19,8 @@ func main() { if path == "" { log.Fatal("path must be set\nusage: checkdb [path]") } - if !*v1 && !*legacy { - log.Fatal("no versions set (use flags -v1 and/or -legacy)") - } - if *v1 { - if _, err := db.Load(path); err != nil { - log.Fatal(err) - } - fmt.Printf("%s contains valid v1 database\n", path) - } else { - fmt.Println("skipping v1 validity check (use flag -v1 to enable)") - } - - if *legacy { - if _, err := legacydb.Load(path); err != nil { - log.Fatal(err) - } - fmt.Printf("%s contains valid legacy database\n", path) - } else { - fmt.Println("skipping legacy validity check (use flag -legacy to enable)") + if _, err := db.Load(path); err != nil { + log.Fatal(err) } + fmt.Printf("%s contains valid v1 database\n", path) } diff --git a/cmd/checkdeploy/main.go b/cmd/checkdeploy/main.go index f62f11e14..8cc5cfeec 100644 --- a/cmd/checkdeploy/main.go +++ b/cmd/checkdeploy/main.go @@ -12,13 +12,11 @@ import ( "log" db "golang.org/x/vulndb/internal/database" - "golang.org/x/vulndb/internal/database/legacydb" ) var ( - newPath = flag.String("new", "", "path to new database") - newLegacyPath = flag.String("legacy", "", "path to the new database in the legacy schema (optional)") - existingPath = flag.String("existing", "", "path to existing database") + newPath = flag.String("new", "", "path to new database") + existingPath = flag.String("existing", "", "path to existing database") ) func main() { @@ -34,16 +32,4 @@ func main() { log.Fatal(err) } fmt.Printf("ok to deploy v1 database %s on top of %s\n", *newPath, *existingPath) - - if *newLegacyPath != "" { - if err := legacydb.Validate(*newLegacyPath, *existingPath); err != nil { - log.Fatal(err) - } - if err := legacydb.Equivalent(*newPath, *newLegacyPath); err != nil { - log.Fatal(err) - } - fmt.Printf("ok to deploy legacy database %s on top of %s\n", *newLegacyPath, *existingPath) - } else { - fmt.Println("not checking legacy database deploy (use -legacy flag to set)") - } } diff --git a/cmd/dbdiff/main.go b/cmd/dbdiff/main.go deleted file mode 100644 index 6f9eecf3a..000000000 --- a/cmd/dbdiff/main.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Command dbdiff provides a tool for comparing two different versions of the -// vulnerability database in legacy format. -package main - -import ( - "fmt" - "log" - "os" - - "golang.org/x/vulndb/internal/database/legacydb" -) - -func main() { - if len(os.Args) != 3 { - fmt.Fprintln(os.Stderr, "usage: dbdiff db-a db-b") - os.Exit(1) - } - if err := legacydb.Diff(os.Args[1], os.Args[2]); err != nil { - log.Fatal(err) - } -} diff --git a/cmd/gendb/main.go b/cmd/gendb/main.go index 9687ab7ef..1c14c9506 100644 --- a/cmd/gendb/main.go +++ b/cmd/gendb/main.go @@ -12,7 +12,6 @@ import ( "log" db "golang.org/x/vulndb/internal/database" - "golang.org/x/vulndb/internal/database/legacydb" "golang.org/x/vulndb/internal/gitrepo" ) @@ -21,7 +20,6 @@ var ( jsonDir = flag.String("out", "out", "Directory to write JSON database to") zipFile = flag.String("zip", "", "if provided, file to write zipped database to (for v1 database only)") indent = flag.Bool("indent", false, "Indent JSON for debugging") - legacy = flag.Bool("legacy", false, "if true, generate in the legacy schema") ) func main() { @@ -31,22 +29,16 @@ func main() { if err != nil { log.Fatal(err) } - if *legacy { - if err := legacydb.Generate(ctx, repo, *jsonDir, *indent); err != nil { - log.Fatal(err) - } - } else { - d, err := db.FromRepo(ctx, repo) - if err != nil { - log.Fatal(err) - } - if err := d.Write(*jsonDir); err != nil { + d, err := db.FromRepo(ctx, repo) + if err != nil { + log.Fatal(err) + } + if err := d.Write(*jsonDir); err != nil { + log.Fatal(err) + } + if *zipFile != "" { + if err := d.WriteZip(*zipFile); err != nil { log.Fatal(err) } - if *zipFile != "" { - if err := d.WriteZip(*zipFile); err != nil { - log.Fatal(err) - } - } } } diff --git a/internal/database/legacydb/database.go b/internal/database/legacydb/database.go deleted file mode 100644 index bfaebe7ad..000000000 --- a/internal/database/legacydb/database.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package legacydb provides functionality for generating, reading, writing, -// and validating vulnerability databases according to the legacy schema. -package legacydb - -import ( - "context" - "encoding/json" - "fmt" - "path" - "time" - - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing/object" - "golang.org/x/vulndb/internal/derrors" - "golang.org/x/vulndb/internal/gitrepo" - "golang.org/x/vulndb/internal/osv" - "golang.org/x/vulndb/internal/report" -) - -// Database is an in-memory representation of a Go vulnerability database, -// following the legacy specification at -// https://go.dev/security/vuln/database#api. -type Database struct { - // A map from module names to the last modified time. - // Represents $dbPath/index.json - Index DBIndex - // Map from each Go ID to its OSV entry. - // Represents $dbPath/ID/index.json and the contents of $dbPath/ID/ - EntriesByID EntriesByID - // Map from each module path to a list of corresponding OSV entries. - // Each map entry represents the contents of a $dbPath/$modulePath.json - // file. - EntriesByModule EntriesByModule - // Map from each alias (CVE and GHSA) ID to a list of Go IDs for that - // alias. - // Represents $dbPath/aliases.json - IDsByAlias IDsByAlias -} - -type ( - DBIndex map[string]time.Time - EntriesByID map[string]*osv.Entry - EntriesByModule map[string][]*osv.Entry - IDsByAlias map[string][]string -) - -const ( - // indexFile is the name of the file that contains the database - // index. - indexFile = "index.json" - - // aliasesFile is the name of the file that contains the database - // aliases index. - aliasesFile = "aliases.json" - - // idDirectory is the name of the directory that contains entries - // listed by their IDs. - idDirectory = "ID" -) - -// New creates a new Database based on the contents of the "data/osv" -// folder in the given repo. -// -// It reads each OSV file, marshals it into a struct, updates the -// modified and published times based on the time of latest and first -// CL to modify the file, and stores the struct in the Database (and updates -// associated index maps). The result is an in-memory vulnerability database -// that can be written to files via Database.Write. -// -// The repo must contain a "data/osv" folder with files in -// OSV JSON format with filenames of the form GO-YYYY-XXXX.json. -// -// New does not modify the repo. -func New(ctx context.Context, repo *git.Repository) (_ *Database, err error) { - defer derrors.Wrap(&err, "New()") - - d := newEmpty() - - root, err := gitrepo.Root(repo) - if err != nil { - return nil, err - } - - commitDates, err := gitrepo.AllCommitDates(repo, gitrepo.HeadReference, report.OSVDir) - if err != nil { - return nil, err - } - - if err = root.Files().ForEach(func(f *object.File) error { - if path.Dir(f.Name) != report.OSVDir || - path.Ext(f.Name) != ".json" { - return nil - } - - // Read the entry. - contents, err := f.Contents() - if err != nil { - return fmt.Errorf("could not read contents of file %s: %v", f.Name, err) - } - var entry osv.Entry - err = json.Unmarshal([]byte(contents), &entry) - if err != nil { - return err - } - - // Set the modified and published times. - dates, ok := commitDates[f.Name] - if !ok { - return fmt.Errorf("can't find git repo commit dates for %q", f.Name) - } - addTimestamps(&entry, dates) - - d.addEntry(&entry) - - return nil - }); err != nil { - return nil, err - } - - return d, nil -} - -func newEmpty() *Database { - return &Database{ - Index: make(DBIndex), - EntriesByID: make(EntriesByID), - EntriesByModule: make(EntriesByModule), - IDsByAlias: make(IDsByAlias), - } -} - -func (d *Database) addEntry(entry *osv.Entry) { - for _, module := range report.ModulesForEntry(*entry) { - d.EntriesByModule[module] = append(d.EntriesByModule[module], entry) - if entry.Modified.After(d.Index[module]) { - d.Index[module] = entry.Modified.Time - } - } - d.EntriesByID[entry.ID] = entry - for _, alias := range entry.Aliases { - d.IDsByAlias[alias] = append(d.IDsByAlias[alias], entry.ID) - } -} - -func addTimestamps(entry *osv.Entry, dates gitrepo.Dates) { - // If a report contains a published field, consider it - // the authoritative source of truth. - // Otherwise, use the time of the earliest commit in the git history. - if entry.Published.IsZero() { - entry.Published = osv.Time{Time: dates.Oldest} - } - - // The modified time is the time of the latest commit for the file. - entry.Modified = osv.Time{Time: dates.Newest} -} diff --git a/internal/database/legacydb/database_test.go b/internal/database/legacydb/database_test.go deleted file mode 100644 index 5b4011fda..000000000 --- a/internal/database/legacydb/database_test.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "context" - "flag" - "testing" - - "github.com/google/go-cmp/cmp" - "golang.org/x/vulndb/internal/gitrepo" -) - -var ( - integration = flag.Bool("integration", false, "test with respect to current contents of vulndb") - - testRepoDir = "testdata/repo.txtar" -) - -func TestNew(t *testing.T) { - ctx := context.Background() - testRepo, err := gitrepo.ReadTxtarRepo(testRepoDir, jan2002.Time) - if err != nil { - t.Fatal(err) - } - got, err := New(ctx, testRepo) - if err != nil { - t.Fatal(err) - } - want := valid - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("unexpected diff (want-, got+):\n%s", diff) - } -} - -func TestAll(t *testing.T) { - ctx := context.Background() - testRepo, err := gitrepo.ReadTxtarRepo(testRepoDir, jan2002.Time) - if err != nil { - t.Fatal(err) - } - new, err := New(ctx, testRepo) - if err != nil { - t.Fatal(err) - } - - writeDir := t.TempDir() - if err = new.Write(writeDir, true); err != nil { - t.Fatal(err) - } - if err = cmpDirHashes(validDir, writeDir); err != nil { - t.Error(err) - } - - validated, err := Load(writeDir) - if err != nil { - t.Error(err) - } - - if diff := cmp.Diff(validated, new); diff != "" { - t.Errorf("unexpected diff (validated-, new+):\n%s", diff) - } -} - -func TestAllIntegration(t *testing.T) { - if !*integration { - t.Skip("Skipping integration tests, use flag -integration to run") - } - - ctx := context.Background() - - repo, err := gitrepo.Open(ctx, "../..") - if err != nil { - t.Fatal(err) - } - new, err := New(ctx, repo) - if err != nil { - t.Fatal(err) - } - - writeDir := t.TempDir() - if err = new.Write(writeDir, true); err != nil { - t.Fatal(err) - } - - validated, err := Load(writeDir) - if err != nil { - t.Error(err) - } - - if diff := cmp.Diff(validated, new); diff != "" { - t.Errorf("unexpected diff (validated-, new+):\n%s", diff) - } -} diff --git a/internal/database/legacydb/diff.go b/internal/database/legacydb/diff.go deleted file mode 100644 index 2749dc938..000000000 --- a/internal/database/legacydb/diff.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "fmt" - - "github.com/google/go-cmp/cmp" - "golang.org/x/vulndb/internal/derrors" -) - -func Diff(dbname1, dbname2 string) (err error) { - defer derrors.Wrap(&err, "Diff(%q, %q)", dbname1, dbname2) - db1, err := Load(dbname1) - if err != nil { - return err - } - db2, err := Load(dbname2) - if err != nil { - return err - } - diff := cmp.Diff(db1, db2) - if diff == "" { - diff = "(no change)" - } - fmt.Printf("diff (-db1, +db2):\n%s", diff) - return nil -} diff --git a/internal/database/legacydb/equivalent.go b/internal/database/legacydb/equivalent.go deleted file mode 100644 index 22cc95b74..000000000 --- a/internal/database/legacydb/equivalent.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "fmt" - - "github.com/google/go-cmp/cmp" - db "golang.org/x/vulndb/internal/database" -) - -// Equivalent returns an error if the v1 database in path does not -// represent the same data as the legacy database in legacyPath, -// or if either database is internally inconsistent according to its -// schema. -func Equivalent(path, legacyPath string) error { - legacy, err := Load(legacyPath) - if err != nil { - return err - } - - v1, err := db.Load(path) - if err != nil { - return err - } - - // Since Load already checks each DB for internal consistency, - // it is sufficient to check that each DB contains the same - // vulnerabilities (OSV entries) and modules. - return legacy.checkSameModulesAndVulns(v1) -} - -func (legacy *Database) checkSameModulesAndVulns(v1 *db.Database) error { - // Check that all the OSV entries are the same. - if v0, v1 := len(legacy.EntriesByID), len(v1.Vulns); v0 != v1 { - return fmt.Errorf("legacy database (num=%d) and v1 database (num=%d) have a different number of vulns", v0, v1) - } - for _, entry := range v1.Entries { - legacyEntry, ok := legacy.EntriesByID[entry.ID] - if !ok { - return fmt.Errorf("v1 database contains vuln %q not present in legacy database", entry.ID) - } - if diff := cmp.Diff(legacyEntry, &entry); diff != "" { - return fmt.Errorf("databases contain a different entry for id %s:\n%s", entry.ID, diff) - } - } - - // Check that all the module paths are the same. - // This is technically not necessary if the validation in Load works - // correctly, but we are double-checking because search-by-module - // is such a common use case. - if m0, m1 := len(legacy.Index), len(v1.Modules); m0 != m1 { - return fmt.Errorf("legacy database (num=%d) and v1 database (num=%d) have a different number of modules", m0, m1) - } - for modulePath := range v1.Modules { - _, ok := legacy.EntriesByModule[modulePath] - if !ok { - return fmt.Errorf("v1 database contains module %q not present in legacy database", modulePath) - } - } - - return nil -} diff --git a/internal/database/legacydb/equivalent_test.go b/internal/database/legacydb/equivalent_test.go deleted file mode 100644 index 34a68f94f..000000000 --- a/internal/database/legacydb/equivalent_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "context" - "testing" - - db "golang.org/x/vulndb/internal/database" - "golang.org/x/vulndb/internal/gitrepo" -) - -func TestEquivalent(t *testing.T) { - ctx := context.Background() - testRepo, err := gitrepo.ReadTxtarRepo(testRepoDir, jan2002.Time) - if err != nil { - t.Fatal(err) - } - - legacyDir := t.TempDir() - err = Generate(ctx, testRepo, legacyDir, true) - if err != nil { - t.Fatal(err) - } - - v1Dir := t.TempDir() - v1, err := db.FromRepo(ctx, testRepo) - if err != nil { - t.Fatal(err) - } - if err := v1.Write(v1Dir); err != nil { - t.Fatal(err) - } - - // Databases created from the same repo should be equivalent. - if err := Equivalent(v1Dir, legacyDir); err != nil { - t.Error(err) - } - - // Equivalent should error because neither of the databases is - // valid according to its schema. - if err := Equivalent(legacyDir, v1Dir); err == nil { - t.Error("Equivalent: got nil, want error") - } -} - -func TestCheckSameModulesAndVulns(t *testing.T) { - legacy := newTestDB(testOSV1, testOSV2, testOSV3) - v1, err := db.New(*testOSV1, *testOSV2) - if err != nil { - t.Fatal(err) - } - - // Should error because legacy DB has an extra entry. - if err := legacy.checkSameModulesAndVulns(v1); err == nil { - t.Error("checkSameModulesAndVulns: got nil, want error") - } - - if err := v1.Add(*testOSV3); err != nil { - t.Error(err) - } - - // Should now succeed. - if err := legacy.checkSameModulesAndVulns(v1); err != nil { - t.Error(err) - } -} diff --git a/internal/database/legacydb/generate.go b/internal/database/legacydb/generate.go deleted file mode 100644 index 02b25fc5d..000000000 --- a/internal/database/legacydb/generate.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "context" - - "github.com/go-git/go-git/v5" - "golang.org/x/vulndb/internal/derrors" -) - -// Generate creates and writes a new Go vulnerability database to outDir -// based on the contents of the "data/osv" folder inside repo. -// -// The repo must contain a "data/osv" folder with files in -// OSV JSON format with filenames of the form GO-YYYY-XXXX.json. -func Generate(ctx context.Context, repo *git.Repository, outDir string, indent bool) (err error) { - defer derrors.Wrap(&err, "Generate()") - - new, err := New(ctx, repo) - if err != nil { - return err - } - if err = new.Write(outDir, indent); err != nil { - return err - } - - return nil -} diff --git a/internal/database/legacydb/generate_test.go b/internal/database/legacydb/generate_test.go deleted file mode 100644 index 55671ab2f..000000000 --- a/internal/database/legacydb/generate_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "context" - "os" - "testing" - - "github.com/google/go-cmp/cmp" - "golang.org/x/vulndb/internal/gitrepo" -) - -func TestGenerate(t *testing.T) { - ctx := context.Background() - testRepo, err := gitrepo.ReadTxtarRepo(testRepoDir, jan2002.Time) - if err != nil { - t.Fatal(err) - } - tmpDir := t.TempDir() - err = Generate(ctx, testRepo, tmpDir, true) - if err != nil { - t.Fatal(err) - } - if err = cmpDirHashes(tmpDir, validDir); err != nil { - t.Error(err) - } -} - -func TestGenerateIntegration(t *testing.T) { - if !*integration { - t.Skip("Skipping integration tests, use flag -integration to run") - } - - moveToVulnDBRoot(t) - - ctx := context.Background() - - repo, err := gitrepo.Open(ctx, ".") - if err != nil { - t.Fatal(err) - } - - genDir := t.TempDir() - err = Generate(ctx, repo, genDir, false) - if err != nil { - t.Fatal(err) - } - - new, err := New(ctx, repo) - if err != nil { - t.Fatal(err) - } - - t.Run("Generate equivalent to New then Write", func(t *testing.T) { - writeDir := t.TempDir() - if err = new.Write(writeDir, false); err != nil { - t.Fatal(err) - } - if err = cmpDirHashes(genDir, writeDir); err != nil { - t.Error(err) - } - }) - - t.Run("New equivalent to Generate then Load", func(t *testing.T) { - loaded, err := Load(genDir) - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(loaded, new); diff != "" { - t.Errorf("unexpected diff (loaded-, new+):\n%s", diff) - } - }) -} - -func moveToVulnDBRoot(t *testing.T) { - // Store current working directory and move into vulndb/ folder. - wd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - if err := os.Chdir("../.."); err != nil { - t.Fatal(err) - } - - // Restore state from before test. - t.Cleanup(func() { - if err = os.Chdir(wd); err != nil { - t.Log(err) - } - }) -} diff --git a/internal/database/legacydb/load.go b/internal/database/legacydb/load.go deleted file mode 100644 index 0134f3d6c..000000000 --- a/internal/database/legacydb/load.go +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "fmt" - "io/fs" - "path/filepath" - "reflect" - "strings" - - "golang.org/x/exp/maps" - "golang.org/x/exp/slices" - dbv1 "golang.org/x/vulndb/internal/database" - "golang.org/x/vulndb/internal/derrors" - "golang.org/x/vulndb/internal/osv" - "golang.org/x/vulndb/internal/report" -) - -// Load reads the contents of dbPath into a Database, and errors if: -// - Any files are malformed (cannot be unmarshaled) -// - The database has missing files (based on the module and ID indexes) -// - The database has unexpected files not listed in the indexes -// - The database is internally inconsistent -func Load(dbPath string) (_ *Database, err error) { - defer derrors.Wrap(&err, "Load(%s)", dbPath) - - d, err := rawLoad(dbPath) - if err != nil { - return nil, err - } - if err := d.checkNoUnexpectedFiles(dbPath); err != nil { - return nil, err - } - if err := d.checkInternalConsistency(); err != nil { - return nil, err - } - - return d, nil -} - -// rawLoad reads the contents of dbPath into a Database, and errors -// if any files are malformed, or the database has missing files -// (based on the module and ID indexes). -func rawLoad(dbPath string) (_ *Database, err error) { - defer derrors.Wrap(&err, "loading data") - - d := &Database{ - Index: make(DBIndex), - IDsByAlias: make(IDsByAlias), - } - - if err := report.UnmarshalFromFile(filepath.Join(dbPath, indexFile), &d.Index); err != nil { - return nil, fmt.Errorf("invalid or missing index.json: %v", err) - } - - d.EntriesByModule, err = loadEntriesByModule(dbPath, d.Index) - if err != nil { - return nil, err - } - - d.EntriesByID, err = loadEntriesByID(dbPath) - if err != nil { - return nil, err - } - - if err := report.UnmarshalFromFile(filepath.Join(dbPath, aliasesFile), &d.IDsByAlias); err != nil { - return nil, fmt.Errorf("invalid or missing aliases.json: %v", err) - } - - return d, nil -} - -func (d *Database) checkNoUnexpectedFiles(dbPath string) error { - return filepath.WalkDir(dbPath, func(path string, f fs.DirEntry, err error) error { - if err != nil { - return err - } - - fname := f.Name() - ext := filepath.Ext(fname) - dir := filepath.Dir(path) - - switch { - // Skip directories. - case f.IsDir(): - return nil - // Skip files in the v1 spec. - case ext == ".gz" || ext == ".zip" || dbv1.IsIndexEndpoint(fname): - return nil - // In the top-level directory, web files and index files are OK. - case dir == dbPath && isIndexOrWebFile(fname, ext): - return nil - // All non-directory and non-web files should end in ".json". - case ext != ".json": - return fmt.Errorf("found unexpected non-JSON file %s", path) - // All files in the ID directory (except the index) should have - // corresponding entries in EntriesByID. - case dir == filepath.Join(dbPath, idDirectory): - if fname == indexFile { - return nil - } - id := report.GoID(fname) - if _, ok := d.EntriesByID[id]; !ok { - return fmt.Errorf("found unexpected file %q which is not present in %s", fname, filepath.Join(idDirectory, indexFile)) - } - // All other files should have corresponding entries in - // EntriesByModule. - default: - module := strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(path, dbPath), string(filepath.Separator)), ".json") - unescaped, err := unescapeModulePath(filepath.ToSlash(module)) - if err != nil { - return fmt.Errorf("could not unescape module file %s: %v", path, err) - } - if _, ok := d.EntriesByModule[unescaped]; !ok { - return fmt.Errorf("found unexpected module %q which is not present in %s", unescaped, indexFile) - } - } - return nil - }) -} - -func isIndexOrWebFile(filename, ext string) bool { - return ext == ".ico" || - ext == ".html" || - // HTML files may have no extension. - ext == "" || - filename == indexFile || - filename == aliasesFile -} - -func (d *Database) checkInternalConsistency() error { - if il, ml := len(d.Index), len(d.EntriesByModule); il != ml { - return fmt.Errorf("length mismatch: there are %d module entries in the index, and %d module directory entries", il, ml) - } - - for module, modified := range d.Index { - entries, ok := d.EntriesByModule[module] - if !ok || len(entries) == 0 { - return fmt.Errorf("no module directory found for indexed module %s", module) - } - - var wantModified osv.Time - for _, entry := range entries { - if mod := entry.Modified; mod.After(wantModified.Time) { - wantModified = mod - } - - entryByID, ok := d.EntriesByID[entry.ID] - if !ok { - return fmt.Errorf("no advisory found for ID %s listed in %s", entry.ID, module) - } - if !reflect.DeepEqual(entry, entryByID) { - return fmt.Errorf("inconsistent OSV contents in module and ID advisory for %s", entry.ID) - } - - var found bool - for _, affected := range entry.Affected { - m := affected.Module.Path - if m == module { - found = true - break - } - } - if !found { - return fmt.Errorf("%s does not reference %s", entry.ID, module) - } - } - if modified != wantModified.Time { - return fmt.Errorf("incorrect modified timestamp for module %s: want %s, got %s", module, wantModified.Time, modified) - } - } - - for id, entry := range d.EntriesByID { - for _, affected := range entry.Affected { - module := affected.Module.Path - entries, ok := d.EntriesByModule[module] - if !ok || len(entries) == 0 { - return fmt.Errorf("module %s not found (referenced by %s)", module, id) - } - found := false - for _, gotEntry := range entries { - if gotEntry.ID == id { - found = true - break - } - } - if !found { - return fmt.Errorf("%s does not have an entry in %s", id, module) - } - } - for _, alias := range entry.Aliases { - gotEntries, ok := d.IDsByAlias[alias] - if !ok || len(gotEntries) == 0 { - return fmt.Errorf("alias %s not found in aliases.json (alias of %s)", alias, id) - } - found := false - for _, gotID := range gotEntries { - if gotID == id { - found = true - break - } - } - if !found { - return fmt.Errorf("%s is not listed as an alias of %s in aliases.json", entry.ID, alias) - } - } - if entry.Published.After(entry.Modified.Time) { - return fmt.Errorf("%s: published time (%s) cannot be after modified time (%s)", entry.ID, entry.Published, entry.Modified) - } - } - - for alias, goIDs := range d.IDsByAlias { - for _, goID := range goIDs { - entry, ok := d.EntriesByID[goID] - if !ok { - return fmt.Errorf("no advisory found for %s listed under %s", goID, alias) - } - - if !slices.Contains(entry.Aliases, alias) { - return fmt.Errorf("advisory %s does not reference alias %s", goID, alias) - } - } - } - - return nil -} -func loadEntriesByID(dbPath string) (EntriesByID, error) { - var ids []string - if err := report.UnmarshalFromFile(filepath.Join(dbPath, idDirectory, indexFile), &ids); err != nil { - return nil, fmt.Errorf("invalid or missing ID/index.json: %v", err) - } - - entriesByID := make(EntriesByID, len(ids)) - for _, id := range ids { - var entry osv.Entry - err := report.UnmarshalFromFile(filepath.Join(dbPath, idDirectory, id+".json"), &entry) - if err != nil { - return nil, fmt.Errorf("invalid or missing OSV file: %v", err) - } - entriesByID[id] = &entry - } - return entriesByID, nil -} - -func loadEntriesByModule(dbPath string, index DBIndex) (EntriesByModule, error) { - entriesByModule := make(EntriesByModule, len(index)) - for _, module := range maps.Keys(index) { - emodule, err := escapeModulePath(module) - if err != nil { - return nil, err - } - fpath := filepath.Join(dbPath, emodule+".json") - var entries []*osv.Entry - err = report.UnmarshalFromFile(fpath, &entries) - if err != nil { - return nil, fmt.Errorf("invalid or missing module directory: %v", err) - } - entriesByModule[module] = entries - } - return entriesByModule, nil -} diff --git a/internal/database/legacydb/load_test.go b/internal/database/legacydb/load_test.go deleted file mode 100644 index d57b5ab06..000000000 --- a/internal/database/legacydb/load_test.go +++ /dev/null @@ -1,343 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "path/filepath" - "strings" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "golang.org/x/vulndb/internal/osv" -) - -var ( - validDir = filepath.FromSlash("testdata/db/valid") - jan1999 = osv.Time{Time: time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC)} - jan2000 = osv.Time{Time: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)} - jan2002 = osv.Time{Time: time.Date(2002, 1, 1, 0, 0, 0, 0, time.UTC)} - testOSV1 = &osv.Entry{ - SchemaVersion: "1.3.1", - ID: "GO-1999-0001", - Published: jan1999, - Modified: jan2002, - Aliases: []string{"CVE-1999-1111"}, - Summary: "A summary", - Details: "Some details", - Affected: []osv.Affected{ - { - Module: osv.Module{ - Path: "example.com/module", - Ecosystem: "Go", - }, - Ranges: []osv.Range{ - { - Type: "SEMVER", - Events: []osv.RangeEvent{ - {Introduced: "0"}, {Fixed: "1.1.0"}, - {Introduced: "1.2.0"}, - {Fixed: "1.2.2"}, - }}}, - EcosystemSpecific: &osv.EcosystemSpecific{ - Packages: []osv.Package{{Path: "example.com/module/package", Symbols: []string{"Symbol"}}}}}, - }, - References: []osv.Reference{ - {Type: "FIX", URL: "https://example.com/cl/123"}, - }, - DatabaseSpecific: &osv.DatabaseSpecific{ - URL: "https://pkg.go.dev/vuln/GO-1999-0001"}} - testOSV2 = &osv.Entry{ - SchemaVersion: "1.3.1", - ID: "GO-2000-0002", - Published: jan2000, - Modified: jan2002, - Aliases: []string{"CVE-1999-2222"}, - Summary: "A summary", - Details: "Some details", - Affected: []osv.Affected{ - { - Module: osv.Module{ - Path: "example.com/module2", - Ecosystem: "Go", - }, - Ranges: []osv.Range{ - { - Type: "SEMVER", Events: []osv.RangeEvent{{Introduced: "0"}, - {Fixed: "1.2.0"}, - }}}, - EcosystemSpecific: &osv.EcosystemSpecific{ - Packages: []osv.Package{{Path: "example.com/module2/package", - Symbols: []string{"Symbol"}, - }}}}}, - References: []osv.Reference{ - {Type: "FIX", URL: "https://example.com/cl/543"}, - }, - DatabaseSpecific: &osv.DatabaseSpecific{URL: "https://pkg.go.dev/vuln/GO-2000-0002"}} - testOSV3 = &osv.Entry{ - SchemaVersion: "1.3.1", - ID: "GO-2000-0003", - Published: jan2002, - Modified: jan2002, - Aliases: []string{"CVE-1999-3333", "GHSA-xxxx-yyyy-zzzz"}, - Summary: "A summary", - Details: "Some details", - Affected: []osv.Affected{ - { - Module: osv.Module{ - Path: "example.com/module2", - Ecosystem: "Go", - }, - Ranges: []osv.Range{ - { - Type: "SEMVER", - Events: []osv.RangeEvent{ - {Introduced: "0"}, {Fixed: "1.1.0"}, - }}}, - EcosystemSpecific: &osv.EcosystemSpecific{Packages: []osv.Package{ - { - Path: "example.com/module2/package", - Symbols: []string{"Symbol"}, - }}}}}, - References: []osv.Reference{ - {Type: "FIX", URL: "https://example.com/cl/000"}, - }, DatabaseSpecific: &osv.DatabaseSpecific{ - URL: "https://pkg.go.dev/vuln/GO-2000-0003", - }} -) - -var valid = &Database{ - Index: DBIndex{ - "example.com/module": jan2002.Time, - "example.com/module2": jan2002.Time, - }, - EntriesByID: EntriesByID{"GO-1999-0001": testOSV1, "GO-2000-0002": testOSV2, "GO-2000-0003": testOSV3}, - EntriesByModule: EntriesByModule{ - "example.com/module": {testOSV1}, - "example.com/module2": {testOSV2, testOSV3}, - }, - IDsByAlias: IDsByAlias{ - "CVE-1999-1111": {"GO-1999-0001"}, - "CVE-1999-2222": {"GO-2000-0002"}, - "CVE-1999-3333": {"GO-2000-0003"}, - "GHSA-xxxx-yyyy-zzzz": {"GO-2000-0003"}, - }, -} - -func TestLoad(t *testing.T) { - t.Run("ok", func(t *testing.T) { - path := validDir - got, err := Load(path) - if err != nil { - t.Fatalf("Load(%s): want success, got %s", path, err) - } - want := valid - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("Load(%s): unexpected diff (want- got+):\n %s", path, diff) - } - }) - - failTests := []struct { - name string - dbPath string - wantErr string - }{ - { - name: "missing file", - dbPath: "testdata/db/missing-file", - wantErr: "invalid or missing", - }, - { - name: "unexpected file", - dbPath: "testdata/db/unexpected-file", - wantErr: "found unexpected file", - }, - } - for _, test := range failTests { - t.Run(test.name, func(t *testing.T) { - _, err := Load(test.dbPath) - if err == nil || !strings.Contains(err.Error(), test.wantErr) { - t.Fatalf("Load(%s): want err containing %s, got %v", test.dbPath, test.wantErr, err) - } - }) - } -} - -func TestCheckInternalConsistency(t *testing.T) { - t.Run("ok", func(t *testing.T) { - if err := valid.checkInternalConsistency(); err != nil { - t.Error(err) - } - }) - - failTests := []struct { - name string - db *Database - wantErr string - }{ - { - name: "too many modules", - db: &Database{ - EntriesByModule: EntriesByModule{"module": []*osv.Entry{}}, - }, - wantErr: "length mismatch", - }, - { - name: "missing module from index", - db: &Database{ - Index: DBIndex{"module": time.Time{}}, - EntriesByModule: EntriesByModule{"module2": []*osv.Entry{}}, - }, - wantErr: "no module directory found", - }, - { - name: "missing OSV from module reference", - db: &Database{ - Index: DBIndex{"module": time.Time{}}, - EntriesByModule: EntriesByModule{"module": []*osv.Entry{ - {ID: "GO-1999-0001"}, - }}, - EntriesByID: EntriesByID{}, - }, - wantErr: "no advisory found for ID GO-1999-0001", - }, - { - name: "inconsistent OSV", - db: &Database{ - Index: DBIndex{"module": time.Time{}}, - EntriesByModule: EntriesByModule{"module": []*osv.Entry{ - {ID: "GO-1999-0001"}, - }}, - EntriesByID: EntriesByID{"GO-1999-0001": {ID: "GO-1999-0001", - Published: jan1999}}, - }, - wantErr: "inconsistent OSV contents", - }, - { - name: "incorrect modified timestamp in index", - db: &Database{ - Index: DBIndex{"module": jan2000.Time}, - EntriesByModule: EntriesByModule{"module": []*osv.Entry{ - {ID: "GO-1999-0001", Modified: jan1999, - Affected: []osv.Affected{ - { - Module: osv.Module{ - Path: "module", - }, - }, - }}, - }}, - EntriesByID: EntriesByID{"GO-1999-0001": {ID: "GO-1999-0001", - Modified: jan1999, Affected: []osv.Affected{ - { - Module: osv.Module{ - Path: "module", - }, - }, - }}}, - }, - wantErr: "incorrect modified timestamp", - }, - { - name: "missing module referenced by OSV", - db: &Database{ - Index: DBIndex{}, - EntriesByModule: EntriesByModule{}, - EntriesByID: EntriesByID{"GO-1999-0001": {ID: "GO-1999-0001", - Affected: []osv.Affected{ - { - Module: osv.Module{ - Path: "a/module", - }, - }, - }, - }}}, - wantErr: "module a/module not found", - }, - { - name: "OSV does not reference module", - db: &Database{ - Index: DBIndex{"module": time.Time{}}, - EntriesByModule: EntriesByModule{"module": []*osv.Entry{ - {ID: "GO-1999-0001"}, - }}, - EntriesByID: EntriesByID{"GO-1999-0001": {ID: "GO-1999-0001"}}, - }, - wantErr: "GO-1999-0001 does not reference module", - }, - { - name: "missing OSV entry in module", - db: &Database{ - Index: DBIndex{"module": time.Time{}}, - EntriesByModule: EntriesByModule{"module": []*osv.Entry{ - {ID: "GO-1999-0002", - Affected: []osv.Affected{ - { - Module: osv.Module{ - Path: "module", - }, - }, - }, - }}}, - EntriesByID: EntriesByID{"GO-1999-0001": {ID: "GO-1999-0001", - Affected: []osv.Affected{ - { - Module: osv.Module{ - Path: "module", - }, - }, - }, - }, "GO-1999-0002": {ID: "GO-1999-0002", - Affected: []osv.Affected{ - { - Module: osv.Module{ - Path: "module", - }, - }, - }, - }, - }}, - wantErr: "GO-1999-0001 does not have an entry in module", - }, - { - name: "missing alias in aliases.json", - db: &Database{ - EntriesByID: EntriesByID{"GO-1999-0001": {ID: "GO-1999-0001", Aliases: []string{"CVE-1999-0001"}}}, - IDsByAlias: IDsByAlias{}, - }, - wantErr: "alias CVE-1999-0001 not found", - }, - { - name: "missing OSV reference in aliases.json", - db: &Database{ - EntriesByID: EntriesByID{"GO-1999-0001": {ID: "GO-1999-0001", Aliases: []string{"CVE-1999-0001"}}}, - IDsByAlias: IDsByAlias{"CVE-1999-0001": []string{"GO-2000-2222"}}, - }, - wantErr: "GO-1999-0001 is not listed as an alias of CVE-1999-0001", - }, - { - name: "missing OSV referenced by aliases.json", - db: &Database{ - IDsByAlias: IDsByAlias{"CVE-1999-0001": []string{"GO-1999-0001"}}, - }, - wantErr: "no advisory found for GO-1999-0001 listed under CVE-1999-0001", - }, - { - name: "missing alias in OSV", - db: &Database{ - EntriesByID: EntriesByID{"GO-1999-0001": {ID: "GO-1999-0001"}}, - IDsByAlias: IDsByAlias{"CVE-1999-0001": []string{"GO-1999-0001"}}, - }, - wantErr: "advisory GO-1999-0001 does not reference alias CVE-1999-0001", - }, - } - for _, test := range failTests { - t.Run(test.name, func(t *testing.T) { - if err := test.db.checkInternalConsistency(); err == nil || !strings.Contains(err.Error(), test.wantErr) { - t.Errorf("want error containing %q, got %v", test.wantErr, err) - } - }) - } -} diff --git a/internal/database/legacydb/testdata/db/existing/ID/GO-1999-0001.json b/internal/database/legacydb/testdata/db/existing/ID/GO-1999-0001.json deleted file mode 100644 index 0a96861c1..000000000 --- a/internal/database/legacydb/testdata/db/existing/ID/GO-1999-0001.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.3.1", - "id": "GO-1999-0001", - "modified": "2001-01-01T00:00:00Z", - "published": "1999-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-1111" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - }, - { - "introduced": "1.2.0" - }, - { - "fixed": "1.2.2" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/123" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-1999-0001" - } -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/existing/ID/GO-2000-0002.json b/internal/database/legacydb/testdata/db/existing/ID/GO-2000-0002.json deleted file mode 100644 index afc8d0b42..000000000 --- a/internal/database/legacydb/testdata/db/existing/ID/GO-2000-0002.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "schema_version": "1.3.1", - "id": "GO-2000-0002", - "modified": "2001-01-01T00:00:00Z", - "published": "2000-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-2222" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module2", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.2.0" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module2/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/543" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-2000-0002" - } -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/existing/ID/index.json b/internal/database/legacydb/testdata/db/existing/ID/index.json deleted file mode 100644 index 949e2f215..000000000 --- a/internal/database/legacydb/testdata/db/existing/ID/index.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - "GO-1999-0001", - "GO-2000-0002" -] \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/existing/aliases.json b/internal/database/legacydb/testdata/db/existing/aliases.json deleted file mode 100644 index 05eaf766a..000000000 --- a/internal/database/legacydb/testdata/db/existing/aliases.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "CVE-1999-1111": [ - "GO-1999-0001" - ], - "CVE-1999-2222": [ - "GO-2000-0002" - ] -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/existing/example.com/module.json b/internal/database/legacydb/testdata/db/existing/example.com/module.json deleted file mode 100644 index 77b01b16a..000000000 --- a/internal/database/legacydb/testdata/db/existing/example.com/module.json +++ /dev/null @@ -1,59 +0,0 @@ -[ - { - "schema_version": "1.3.1", - "id": "GO-1999-0001", - "modified": "2001-01-01T00:00:00Z", - "published": "1999-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-1111" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - }, - { - "introduced": "1.2.0" - }, - { - "fixed": "1.2.2" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/123" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-1999-0001" - } - } -] \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/existing/example.com/module2.json b/internal/database/legacydb/testdata/db/existing/example.com/module2.json deleted file mode 100644 index aedcca1bf..000000000 --- a/internal/database/legacydb/testdata/db/existing/example.com/module2.json +++ /dev/null @@ -1,53 +0,0 @@ -[ - { - "schema_version": "1.3.1", - "id": "GO-2000-0002", - "modified": "2001-01-01T00:00:00Z", - "published": "2000-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-2222" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module2", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.2.0" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module2/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/543" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-2000-0002" - } - } -] \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/existing/index.json b/internal/database/legacydb/testdata/db/existing/index.json deleted file mode 100644 index 9fe1aef28..000000000 --- a/internal/database/legacydb/testdata/db/existing/index.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "example.com/module": "2001-01-01T00:00:00Z", - "example.com/module2": "2001-01-01T00:00:00Z" -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/missing-file/ID/GO-1999-0001.json b/internal/database/legacydb/testdata/db/missing-file/ID/GO-1999-0001.json deleted file mode 100644 index 0a96861c1..000000000 --- a/internal/database/legacydb/testdata/db/missing-file/ID/GO-1999-0001.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.3.1", - "id": "GO-1999-0001", - "modified": "2001-01-01T00:00:00Z", - "published": "1999-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-1111" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - }, - { - "introduced": "1.2.0" - }, - { - "fixed": "1.2.2" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/123" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-1999-0001" - } -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/missing-file/ID/GO-2000-0002.json b/internal/database/legacydb/testdata/db/missing-file/ID/GO-2000-0002.json deleted file mode 100644 index afc8d0b42..000000000 --- a/internal/database/legacydb/testdata/db/missing-file/ID/GO-2000-0002.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "schema_version": "1.3.1", - "id": "GO-2000-0002", - "modified": "2001-01-01T00:00:00Z", - "published": "2000-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-2222" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module2", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.2.0" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module2/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/543" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-2000-0002" - } -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/missing-file/ID/index.json b/internal/database/legacydb/testdata/db/missing-file/ID/index.json deleted file mode 100644 index 949e2f215..000000000 --- a/internal/database/legacydb/testdata/db/missing-file/ID/index.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - "GO-1999-0001", - "GO-2000-0002" -] \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/missing-file/aliases.json b/internal/database/legacydb/testdata/db/missing-file/aliases.json deleted file mode 100644 index 05eaf766a..000000000 --- a/internal/database/legacydb/testdata/db/missing-file/aliases.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "CVE-1999-1111": [ - "GO-1999-0001" - ], - "CVE-1999-2222": [ - "GO-2000-0002" - ] -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/missing-file/example.com/module.json b/internal/database/legacydb/testdata/db/missing-file/example.com/module.json deleted file mode 100644 index 28b992cc8..000000000 --- a/internal/database/legacydb/testdata/db/missing-file/example.com/module.json +++ /dev/null @@ -1,59 +0,0 @@ -[ - { - "schema_version": "1.3.1", - "id": "GO-1999-0001", - "modified": "2001-01-01T00:00:00Z", - "published": "1999-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-1111" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - }, - { - "introduced": "1.2.0" - }, - { - "fixed": "1.2.2" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/123" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-1999-0001" - } - } -] \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/missing-file/index.json b/internal/database/legacydb/testdata/db/missing-file/index.json deleted file mode 100644 index 9fe1aef28..000000000 --- a/internal/database/legacydb/testdata/db/missing-file/index.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "example.com/module": "2001-01-01T00:00:00Z", - "example.com/module2": "2001-01-01T00:00:00Z" -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/unexpected-file/ID/GO-1999-0001.json b/internal/database/legacydb/testdata/db/unexpected-file/ID/GO-1999-0001.json deleted file mode 100644 index 19fe79494..000000000 --- a/internal/database/legacydb/testdata/db/unexpected-file/ID/GO-1999-0001.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.3.1", - "id": "GO-1999-0001", - "modified": "2002-01-01T00:00:00Z", - "published": "1999-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-1111" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - }, - { - "introduced": "1.2.0" - }, - { - "fixed": "1.2.2" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/123" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-1999-0001" - } -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/unexpected-file/ID/GO-2000-0002.json b/internal/database/legacydb/testdata/db/unexpected-file/ID/GO-2000-0002.json deleted file mode 100644 index 12b4486f4..000000000 --- a/internal/database/legacydb/testdata/db/unexpected-file/ID/GO-2000-0002.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "schema_version": "1.3.1", - "id": "GO-2000-0002", - "modified": "2002-01-01T00:00:00Z", - "published": "2000-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-2222" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module2", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.2.0" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module2/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/543" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-2000-0002" - } -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/unexpected-file/ID/index.json b/internal/database/legacydb/testdata/db/unexpected-file/ID/index.json deleted file mode 100644 index 286d5c83a..000000000 --- a/internal/database/legacydb/testdata/db/unexpected-file/ID/index.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - "GO-1999-0001" -] \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/unexpected-file/aliases.json b/internal/database/legacydb/testdata/db/unexpected-file/aliases.json deleted file mode 100644 index ab04a2a6f..000000000 --- a/internal/database/legacydb/testdata/db/unexpected-file/aliases.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "CVE-1999-1111": [ - "GO-1999-0001" - ] -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/unexpected-file/example.com/module.json b/internal/database/legacydb/testdata/db/unexpected-file/example.com/module.json deleted file mode 100644 index e393ec11b..000000000 --- a/internal/database/legacydb/testdata/db/unexpected-file/example.com/module.json +++ /dev/null @@ -1,59 +0,0 @@ -[ - { - "schema_version": "1.3.1", - "id": "GO-1999-0001", - "modified": "2002-01-01T00:00:00Z", - "published": "1999-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-1111" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - }, - { - "introduced": "1.2.0" - }, - { - "fixed": "1.2.2" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/123" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-1999-0001" - } - } -] \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/unexpected-file/index.json b/internal/database/legacydb/testdata/db/unexpected-file/index.json deleted file mode 100644 index cbbec59b5..000000000 --- a/internal/database/legacydb/testdata/db/unexpected-file/index.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "example.com/module": "2002-01-01T00:00:00Z" -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/valid/ID/GO-1999-0001.json b/internal/database/legacydb/testdata/db/valid/ID/GO-1999-0001.json deleted file mode 100644 index 19fe79494..000000000 --- a/internal/database/legacydb/testdata/db/valid/ID/GO-1999-0001.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.3.1", - "id": "GO-1999-0001", - "modified": "2002-01-01T00:00:00Z", - "published": "1999-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-1111" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - }, - { - "introduced": "1.2.0" - }, - { - "fixed": "1.2.2" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/123" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-1999-0001" - } -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/valid/ID/GO-2000-0002.json b/internal/database/legacydb/testdata/db/valid/ID/GO-2000-0002.json deleted file mode 100644 index 12b4486f4..000000000 --- a/internal/database/legacydb/testdata/db/valid/ID/GO-2000-0002.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "schema_version": "1.3.1", - "id": "GO-2000-0002", - "modified": "2002-01-01T00:00:00Z", - "published": "2000-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-2222" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module2", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.2.0" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module2/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/543" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-2000-0002" - } -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/valid/ID/GO-2000-0003.json b/internal/database/legacydb/testdata/db/valid/ID/GO-2000-0003.json deleted file mode 100644 index 970939d46..000000000 --- a/internal/database/legacydb/testdata/db/valid/ID/GO-2000-0003.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "schema_version": "1.3.1", - "id": "GO-2000-0003", - "modified": "2002-01-01T00:00:00Z", - "published": "2002-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-3333", - "GHSA-xxxx-yyyy-zzzz" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module2", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module2/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/000" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-2000-0003" - } -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/valid/ID/index.json b/internal/database/legacydb/testdata/db/valid/ID/index.json deleted file mode 100644 index c14dc6475..000000000 --- a/internal/database/legacydb/testdata/db/valid/ID/index.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - "GO-1999-0001", - "GO-2000-0002", - "GO-2000-0003" -] \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/valid/aliases.json b/internal/database/legacydb/testdata/db/valid/aliases.json deleted file mode 100644 index 6eb259b68..000000000 --- a/internal/database/legacydb/testdata/db/valid/aliases.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "CVE-1999-1111": [ - "GO-1999-0001" - ], - "CVE-1999-2222": [ - "GO-2000-0002" - ], - "CVE-1999-3333": [ - "GO-2000-0003" - ], - "GHSA-xxxx-yyyy-zzzz": [ - "GO-2000-0003" - ] -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/valid/example.com/module.json b/internal/database/legacydb/testdata/db/valid/example.com/module.json deleted file mode 100644 index 68dd8b2ba..000000000 --- a/internal/database/legacydb/testdata/db/valid/example.com/module.json +++ /dev/null @@ -1,59 +0,0 @@ -[ - { - "schema_version": "1.3.1", - "id": "GO-1999-0001", - "modified": "2002-01-01T00:00:00Z", - "published": "1999-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-1111" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - }, - { - "introduced": "1.2.0" - }, - { - "fixed": "1.2.2" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/123" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-1999-0001" - } - } -] \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/valid/example.com/module2.json b/internal/database/legacydb/testdata/db/valid/example.com/module2.json deleted file mode 100644 index e0dffe77b..000000000 --- a/internal/database/legacydb/testdata/db/valid/example.com/module2.json +++ /dev/null @@ -1,105 +0,0 @@ -[ - { - "schema_version": "1.3.1", - "id": "GO-2000-0002", - "modified": "2002-01-01T00:00:00Z", - "published": "2000-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-2222" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module2", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.2.0" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module2/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/543" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-2000-0002" - } - }, - { - "schema_version": "1.3.1", - "id": "GO-2000-0003", - "modified": "2002-01-01T00:00:00Z", - "published": "2002-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-3333", - "GHSA-xxxx-yyyy-zzzz" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module2", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module2/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/000" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-2000-0003" - } - } -] \ No newline at end of file diff --git a/internal/database/legacydb/testdata/db/valid/index.json b/internal/database/legacydb/testdata/db/valid/index.json deleted file mode 100644 index a60960caf..000000000 --- a/internal/database/legacydb/testdata/db/valid/index.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "example.com/module": "2002-01-01T00:00:00Z", - "example.com/module2": "2002-01-01T00:00:00Z" -} \ No newline at end of file diff --git a/internal/database/legacydb/testdata/repo.txtar b/internal/database/legacydb/testdata/repo.txtar deleted file mode 100644 index 31d1ab036..000000000 --- a/internal/database/legacydb/testdata/repo.txtar +++ /dev/null @@ -1,167 +0,0 @@ --- data/osv/GO-1999-0001.json -- -{ - "schema_version": "1.3.1", - "id": "GO-1999-0001", - "modified": "0001-01-01T00:00:00Z", - "published": "1999-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-1111" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - }, - { - "introduced": "1.2.0" - }, - { - "fixed": "1.2.2" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/123" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-1999-0001" - } -} - --- data/osv/GO-2000-0002.json -- -{ - "schema_version": "1.3.1", - "id": "GO-2000-0002", - "modified": "0001-01-01T00:00:00Z", - "published": "2000-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-2222" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module2", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.2.0" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module2/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/543" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-2000-0002" - } -} - --- data/osv/GO-2000-0003.json -- -{ - "schema_version": "1.3.1", - "id": "GO-2000-0003", - "modified": "0001-01-01T00:00:00Z", - "published": "0001-01-01T00:00:00Z", - "aliases": [ - "CVE-1999-3333", - "GHSA-xxxx-yyyy-zzzz" - ], - "summary": "A summary", - "details": "Some details", - "affected": [ - { - "package": { - "name": "example.com/module2", - "ecosystem": "Go" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.1.0" - } - ] - } - ], - "ecosystem_specific": { - "imports": [ - { - "path": "example.com/module2/package", - "symbols": [ - "Symbol" - ] - } - ] - } - } - ], - "references": [ - { - "type": "FIX", - "url": "https://example.com/cl/000" - } - ], - "database_specific": { - "url": "https://pkg.go.dev/vuln/GO-2000-0003" - } -} - - diff --git a/internal/database/legacydb/validate.go b/internal/database/legacydb/validate.go deleted file mode 100644 index e77f999b1..000000000 --- a/internal/database/legacydb/validate.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "fmt" - - "golang.org/x/vulndb/internal/derrors" -) - -// Validate checks that the databases in newPath and oldPath are -// both valid databases, and that the database in newPath can -// be safely deployed on top of the database in oldPath. -func Validate(newPath, oldPath string) (err error) { - derrors.Wrap(&err, "Validate(new=%s, old=%s)", newPath, oldPath) - - // Load will fail if either of the databases is internally - // inconsistent. - new, err := Load(newPath) - if err != nil { - return err - } - old, err := Load(oldPath) - if err != nil { - return err - } - - return validate(new, old) -} - -// validate checks for deleted files and inconsistent timestamps. -func validate(new, old *Database) error { - for id, oldEntry := range old.EntriesByID { - newEntry, ok := new.EntriesByID[id] - if !ok { - return fmt.Errorf("%s is not present in new database. Use the %q field to delete an entry", id, "withdrawn") - } - if newEntry.Published != oldEntry.Published { - return fmt.Errorf("%s: published time cannot change (new %s, old %s)", id, newEntry.Published, oldEntry.Published) - } - if newEntry.Modified.Before(oldEntry.Modified.Time) { - return fmt.Errorf("%s: modified time cannot decrease (new %s, old %s)", id, newEntry.Modified, oldEntry.Modified) - } - } - return nil -} diff --git a/internal/database/legacydb/validate_test.go b/internal/database/legacydb/validate_test.go deleted file mode 100644 index f62496b34..000000000 --- a/internal/database/legacydb/validate_test.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "path/filepath" - "strings" - "testing" - - "golang.org/x/vulndb/internal/osv" -) - -var existingDir = filepath.FromSlash("testdata/db/existing") - -func TestValidate(t *testing.T) { - t.Run("valid", func(t *testing.T) { - // validDir contains a new entry and correct modified and - // published timestamps with respect to existing. - if err := Validate(validDir, existingDir); err != nil { - t.Error(err) - } - }) - - t.Run("old on new fails", func(t *testing.T) { - // Attempting to deploy the existing database on top of the new - // database should fail, as an entry is missing and timestamps are - // incorrect. - if err := Validate(existingDir, validDir); err == nil { - t.Error("expected error, got nil") - } - }) -} - -func newTestDB(entries ...*osv.Entry) *Database { - d := newEmpty() - for _, entry := range entries { - d.addEntry(entry) - } - return d -} - -func TestValidateInternal(t *testing.T) { - successTests := []struct { - name string - newDB *Database - oldDB *Database - }{ - { - name: "valid updates ok", - oldDB: newTestDB( - &osv.Entry{ - ID: "GO-1999-0001", - Published: jan1999, - Modified: jan1999, - }), - newDB: newTestDB(&osv.Entry{ - ID: "GO-1999-0001", - Published: jan1999, - Modified: jan2000, - }, &osv.Entry{ - ID: "GO-1999-0002", - Published: jan2000, - Modified: jan2000, - }), - }, - { - name: "same db ok", - oldDB: valid, - newDB: valid, - }, - } - for _, test := range successTests { - t.Run(test.name, func(t *testing.T) { - if err := validate(test.newDB, test.oldDB); err != nil { - t.Errorf("validate(): unexpected error %v", err) - } - }) - } - - failTests := []struct { - name string - newDB *Database - oldDB *Database - wantErr string - }{ - { - name: "published time changed", - oldDB: newTestDB( - &osv.Entry{ - ID: "GO-1999-0001", - Published: jan1999, - Modified: jan1999, - }), - newDB: newTestDB(&osv.Entry{ - ID: "GO-1999-0001", - Published: jan2000, - Modified: jan2000, - }), - wantErr: "published time cannot change", - }, - { - name: "deleted entry", - oldDB: newTestDB( - &osv.Entry{ - ID: "GO-1999-0001", - Published: jan1999, - Modified: jan1999, - }), - newDB: newTestDB(), - wantErr: "GO-1999-0001 is not present in new database", - }, - { - name: "modified time decreased", - oldDB: newTestDB( - &osv.Entry{ - ID: "GO-1999-0001", - Modified: jan2000, - }), - newDB: newTestDB(&osv.Entry{ - ID: "GO-1999-0001", - Modified: jan1999, - }), - wantErr: "modified time cannot decrease", - }, - } - for _, test := range failTests { - t.Run(test.name, func(t *testing.T) { - if err := validate(test.newDB, test.oldDB); err == nil || !strings.Contains(err.Error(), test.wantErr) { - t.Errorf("validate(): want error containing %q, got %v", test.wantErr, err) - } - }) - } -} diff --git a/internal/database/legacydb/write.go b/internal/database/legacydb/write.go deleted file mode 100644 index 9247f437f..000000000 --- a/internal/database/legacydb/write.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "fmt" - "os" - "path/filepath" - - "golang.org/x/exp/maps" - "golang.org/x/exp/slices" - "golang.org/x/mod/module" - db "golang.org/x/vulndb/internal/database" - "golang.org/x/vulndb/internal/derrors" - "golang.org/x/vulndb/internal/osv" -) - -// Write writes the contents of the Database to JSON files, -// following the legacy specification in -// https://go.dev/security/vuln/database#api. -// path is the base path where the database will be written, and indent -// indicates if the JSON should be indented. -func (d *Database) Write(path string, indent bool) (err error) { - defer derrors.Wrap(&err, "Database.Write(%q)", path) - - if err := os.MkdirAll(path, 0755); err != nil { - return fmt.Errorf("failed to create directory %q: %s", path, err) - } - - if err = d.writeIndex(path, indent); err != nil { - return err - } - - if err = d.writeAliasIndex(path, indent); err != nil { - return err - } - - if err = d.writeEntriesByModule(path, indent); err != nil { - return err - } - - if err = d.writeEntriesByID(path, indent); err != nil { - return err - } - - return nil -} - -func (d *Database) writeIndex(path string, indent bool) error { - return db.WriteJSON(filepath.Join(path, indexFile), d.Index, indent) -} - -func (d *Database) writeAliasIndex(path string, indent bool) error { - return db.WriteJSON(filepath.Join(path, aliasesFile), d.IDsByAlias, indent) -} - -func (d *Database) writeEntriesByModule(path string, indent bool) error { - for module, entries := range d.EntriesByModule { - epath, err := escapeModulePath(module) - if err != nil { - return err - } - outPath := filepath.Join(path, epath) - if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { - return fmt.Errorf("failed to create directory %q: %s", filepath.Dir(outPath), err) - } - if err = db.WriteJSON(outPath+".json", entries, indent); err != nil { - return err - } - } - return nil -} - -func (d *Database) writeEntriesByID(path string, indent bool) error { - idDirPath := filepath.Join(path, idDirectory) - if err := os.MkdirAll(idDirPath, 0755); err != nil { - return fmt.Errorf("failed to create directory %q: %v", idDirPath, err) - } - // Write the entry files. - for _, entry := range d.EntriesByID { - if err := db.WriteJSON(filepath.Join(idDirPath, entry.ID+".json"), entry, indent); err != nil { - return err - } - } - idIndex := maps.Keys(d.EntriesByID) - slices.Sort(idIndex) - // Write the ID Index. - return db.WriteJSON(filepath.Join(idDirPath, indexFile), idIndex, indent) -} - -// Pseudo-module paths used for parts of the Go system. -// These are technically not valid module paths, so we -// mustn't pass them to module.EscapePath. -// Keep in sync with vulndb/internal/database/generate.go. -var specialCaseModulePaths = map[string]bool{ - osv.GoStdModulePath: true, - osv.GoCmdModulePath: true, -} - -// escapeModulePath is like golang.org/x/mod/module.EscapePath, -// but accounts for special paths used by the vulnerability database. -func escapeModulePath(path string) (string, error) { - if specialCaseModulePaths[path] { - return path, nil - } - return module.EscapePath(path) -} - -// unescapeModulePath is like golang.org/x/mod/module.UnescapePath, but -// accounts for special paths used by the vulnerability database. -func unescapeModulePath(path string) (string, error) { - if specialCaseModulePaths[path] { - return path, nil - } - return module.UnescapePath(path) -} diff --git a/internal/database/legacydb/write_test.go b/internal/database/legacydb/write_test.go deleted file mode 100644 index e57d1f980..000000000 --- a/internal/database/legacydb/write_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package legacydb - -import ( - "fmt" - "testing" - - "github.com/google/go-cmp/cmp" - "golang.org/x/mod/sumdb/dirhash" -) - -func TestWrite(t *testing.T) { - tempDir := t.TempDir() - - if err := valid.Write(tempDir, true); err != nil { - t.Fatal(err) - } - - if err := cmpDirHashes(tempDir, validDir); err != nil { - t.Error(err) - } -} - -// cmpDirHashes compares the contents of two directories by comparing -// their hashes. -func cmpDirHashes(d1, d2 string) error { - h1, err := dirhash.HashDir(d1, "", dirhash.DefaultHash) - if err != nil { - return fmt.Errorf("could not hash dir %q: %v", d1, err) - } - h2, err := dirhash.HashDir(d2, "", dirhash.DefaultHash) - if err != nil { - return fmt.Errorf("could not hash dir %q: %v", d2, err) - } - if h1 != h2 { - return fmt.Errorf("hashes do not match:\n%s: %s\n%s: %s", d1, h1, d2, h2) - } - return nil -} - -// Check that Write and Load are inverses. -func TestWriteLoad(t *testing.T) { - tempDir := t.TempDir() - - written := valid - if err := written.Write(tempDir, false); err != nil { - t.Fatal(err) - } - loaded, err := Load(tempDir) - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(written, loaded); diff != "" { - t.Errorf("unexpected diff (written- loaded+):\n %s", diff) - } -}