feat(fortinet): new support for fortinet data feed (#336)

.github/workflows/fetch.yml

@@ -125,3 +125,63 @@ jobs:
       - name: fetch redis
         if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
         run: ./go-cve-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" jvn
+
+  fetch-fortinet:
+    name: fetch-fortinet
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mysql
+        ports:
+          - 3306:3306
+        env:
+          MYSQL_ROOT_PASSWORD: password
+          MYSQL_DATABASE: test
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+      - name: build
+        id: build
+        run: make build
+      - name: fetch sqlite3
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./go-cve-dictionary fetch --dbtype sqlite3 fortinet
+      - name: fetch mysql
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./go-cve-dictionary fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" fortinet
+      - name: fetch postgres
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./go-cve-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" fortinet
+      - name: fetch redis
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./go-cve-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" fortinet

README.md

@@ -242,6 +242,7 @@ Usage:
   go-cve-dictionary fetch [command]
 
 Available Commands:
+  fortinet    Fetch Vulnerability dictionary from Fortinet Advisories
   jvn         Fetch Vulnerability dictionary from JVN
   nvd         Fetch Vulnerability dictionary from NVD
 
@@ -284,6 +285,12 @@ $ go-cve-dictionary fetch jvn
 ```bash
 $ go-cve-dictionary fetch jvn 2021
 ```
+
+#### Fetch Fortinet data
+```bash
+$ go-cve-dictionary fetch fortinet
+```
+
 ----
 
 ### Usage: Run HTTP Server
@@ -332,6 +339,14 @@ Global Flags:
           --dbpath "user:pass@tcp(localhost:3306)/dbname?parseTime=true"
     ```
 
+- fetch fortinet
+
+    ```bash
+    $ go-cve-dictionary fetch fortinet \
+          --dbtype mysql \
+          --dbpath "user:pass@tcp(localhost:3306)/dbname?parseTime=true"
+    ```
+
 - server
 
     ```bash
@@ -358,6 +373,14 @@ Global Flags:
           --dbpath "host=myhost user=user dbname=dbname sslmode=disable password=password"
     ```
 
+- fetch fortinet
+
+    ```bash
+    $ go-cve-dictionary fetch fortinet \
+          --dbtype postgres \
+          --dbpath "host=myhost user=user dbname=dbname sslmode=disable password=password"
+    ```
+
 - server
 
     ```bash
@@ -384,6 +407,14 @@ Global Flags:
           --dbpath "redis://localhost/0"
     ```
 
+- fetch fortinet
+
+    ```bash
+    $ go-cve-dictionary fetch fortinet \
+          --dbtype redis \
+          --dbpath "redis://localhost/0"
+    ```
+
 - server
 
     ```bash
@@ -434,6 +465,7 @@ Run with --debug, --debug-sql option.
 
 - [NVD](https://nvd.nist.gov/)
 - [JVN(Japanese)](http://jvndb.jvn.jp/apis/myjvn/)
+- [Fortinet(https://www.fortiguard.com/psirt)](https://github.com/vulsio/vuls-data-raw-fortinet)
 
 ----
 

commands/fetchfortinet.go

@@ -0,0 +1,69 @@
+package commands
+
+import (
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"golang.org/x/xerrors"
+
+	db "github.com/vulsio/go-cve-dictionary/db"
+	"github.com/vulsio/go-cve-dictionary/fetcher/fortinet"
+	log "github.com/vulsio/go-cve-dictionary/log"
+	"github.com/vulsio/go-cve-dictionary/models"
+)
+
+var fetchFortinetCmd = &cobra.Command{
+	Use:   "fortinet",
+	Short: "Fetch Vulnerability dictionary from Fortinet Advisories",
+	Long:  "Fetch Vulnerability dictionary from Fortinet Advisories",
+	RunE:  fetchFortinet,
+}
+
+func init() {
+	fetchCmd.AddCommand(fetchFortinetCmd)
+}
+
+func fetchFortinet(_ *cobra.Command, _ []string) (err error) {
+	if err := log.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+	}
+
+	driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
+	if err != nil {
+		if xerrors.Is(err, db.ErrDBLocked) {
+			return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err)
+		}
+		return xerrors.Errorf("Failed to open DB. err: %w", err)
+	}
+
+	fetchMeta, err := driver.GetFetchMeta()
+	if err != nil {
+		return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err)
+	}
+	if fetchMeta.OutDated() {
+		return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
+	}
+	// If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion.
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	advs, err := fortinet.FetchConvert()
+	if err != nil {
+		return xerrors.Errorf("Failed to fetch from fortinet. err: %w", err)
+	}
+
+	log.Infof("Inserting Fortinet into DB (%s).", driver.Name())
+	if err := driver.InsertFortinet(advs); err != nil {
+		return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	fetchMeta.LastFetchedAt = time.Now()
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	log.Infof("Finished fetching Fortinet.")
+	return nil
+}

commands/server.go

@@ -64,11 +64,19 @@ func executeServer(_ *cobra.Command, _ []string) (err error) {
 	}
 	count += jvnCount
 
+	fortinetCount, err := driver.CountFortinet()
+	if err != nil {
+		log.Errorf("Failed to count Fortinet table: %s", err)
+		return err
+	}
+	count += fortinetCount
+
 	if count == 0 {
-		log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN")
+		log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet")
 		log.Infof("")
 		log.Infof(" go-cve-dictionary fetch nvd")
 		log.Infof(" go-cve-dictionary fetch jvn")
+		log.Infof(" go-cve-dictionary fetch fortinet")
 		log.Infof("")
 		return nil
 	}

db/db.go

@@ -29,12 +29,14 @@ type DB interface {
 
 	Get(string) (*models.CveDetail, error)
 	GetMulti([]string) (map[string]models.CveDetail, error)
-	GetCveIDsByCpeURI(string) ([]string, []string, error)
+	GetCveIDsByCpeURI(string) ([]string, []string, []string, error)
 	GetByCpeURI(string) ([]models.CveDetail, error)
 	InsertJvn([]string) error
 	InsertNvd([]string) error
+	InsertFortinet([]models.Fortinet) error
 	CountNvd() (int, error)
 	CountJvn() (int, error)
+	CountFortinet() (int, error)
 }
 
 // Option :
@@ -306,45 +308,75 @@ func matchRpmVer(specifiedVer string, cpeInNvd models.CpeBase) bool {
 	return true
 }
 
-func matchCpe(uri string, cve *models.CveDetail) (nvdMatch, jvnMatch bool, err error) {
-	for _, nvd := range cve.Nvds {
-		for _, cpe := range nvd.Cpes {
-			isExactMatch, isRoughMatch, isVendorProductMatch, err := match(uri, cpe.CpeBase)
-			if err != nil {
-				log.Debugf("Failed to match: %s", err)
-				continue
+func matchCpe(uri string, cve *models.CveDetail) (nvdMatch, jvnMatch, fortinetMatch bool, err error) {
+	nvdMatch = false
+	if cve.HasNvd() {
+		for _, nvd := range cve.Nvds {
+			for _, cpe := range nvd.Cpes {
+				isExactMatch, isRoughMatch, isVendorProductMatch, err := match(uri, cpe.CpeBase)
+				if err != nil {
+					log.Debugf("Failed to match: %s", err)
+					continue
+				}
+				if isExactMatch || isRoughMatch || isVendorProductMatch {
+					nvdMatch = true
+					break
+				}
 			}
-			if isExactMatch || isRoughMatch || isVendorProductMatch {
-				return true, false, nil
+			if nvdMatch {
+				break
 			}
 		}
 	}
 
-	if !cve.HasJvn() {
-		return false, false, nil
-	}
-
-	// CPE that exists only in JVN is also detected.
-	// There is a possibility of false positives since the JVN does not contain version information.
-	for _, jvn := range cve.Jvns {
-		for _, jvnCpe := range jvn.Cpes {
-			// If NVD has data of the same `part`, `vendor`, and `product`, NVD is used in priority.
-			// Because NVD has version information, but JVN does not.
-			if isCpeURIAlsoDefinedInNvd(jvnCpe.URI, cve.Nvds) {
-				continue
+	jvnMatch = false
+	if cve.HasJvn() {
+		// CPE that exists only in JVN is also detected.
+		// There is a possibility of false positives since the JVN does not contain version information.
+		for _, jvn := range cve.Jvns {
+			for _, jvnCpe := range jvn.Cpes {
+				// If NVD has data of the same `part`, `vendor`, and `product`, NVD is used in priority.
+				// Because NVD has version information, but JVN does not.
+				if isCpeURIAlsoDefinedInNvd(jvnCpe.URI, cve.Nvds) {
+					continue
+				}
+
+				ok, err := isSamePartVendorProduct(uri, jvnCpe.URI)
+				if err != nil {
+					continue
+				}
+				if ok {
+					jvnMatch = true
+					break
+				}
+			}
+			if jvnMatch {
+				break
 			}
+		}
+	}
 
-			ok, err := isSamePartVendorProduct(uri, jvnCpe.URI)
-			if err != nil {
-				continue
+	fortinetMatch = false
+	if cve.HasFortinet() {
+		for _, fortinet := range cve.Fortinets {
+			for _, cpe := range fortinet.Cpes {
+				isExactMatch, isRoughMatch, isVendorProductMatch, err := match(uri, cpe.CpeBase)
+				if err != nil {
+					log.Debugf("Failed to match: %s", err)
+					continue
+				}
+				if isExactMatch || isRoughMatch || isVendorProductMatch {
+					fortinetMatch = true
+					break
+				}
 			}
-			if ok {
-				return false, true, nil
+			if fortinetMatch {
+				break
 			}
 		}
 	}
 
-	return false, false, nil
+	return nvdMatch, jvnMatch, fortinetMatch, nil
 }
 
 func isSuperORSubset(source, target common.WellFormedName) bool {
@@ -414,6 +446,28 @@ func filterCveDetailByCpeURI(uri string, d *models.CveDetail) error {
 		}
 	}
 
+	fortinets := append([]models.Fortinet{}, d.Fortinets...)
+	d.Fortinets = []models.Fortinet{}
+	for _, fortinet := range fortinets {
+		for _, cpe := range fortinet.Cpes {
+			isExactMatch, isRoughMatch, isVendorProductMatch, err := match(uri, cpe.CpeBase)
+			if err != nil {
+				log.Debugf("Failed to match. err: %s, uri: %s, CpeBase: %#v", err, uri, cpe.CpeBase)
+			}
+			if isExactMatch {
+				fortinet.DetectionMethod = models.FortinetExactVersionMatch
+			} else if isRoughMatch {
+				fortinet.DetectionMethod = models.FortinetRoughVersionMatch
+			} else if isVendorProductMatch {
+				fortinet.DetectionMethod = models.FortinetVendorProductMatch
+			}
+			if isExactMatch || isRoughMatch || isVendorProductMatch {
+				d.Fortinets = append(d.Fortinets, fortinet)
+				break
+			}
+		}
+	}
+
 	return nil
 }