feat: change fetch command (#214)

* feat: change fetch command * feat(fetcher): update fetcher * chore(rdb): remove unnecessary func * chore: go mod tidy * chore: update fetch command * fix(redis): avoid i/o timeout * chore(integration): output JSON when diffs exist * chore: add comment * docs: update README * docs: fix typo
vulsio · Sep 13, 2021 · 6c93275 · 6c93275
1 parent 6992249
commit 6c93275
Show file tree

Hide file tree

Showing 19 changed files with 436 additions and 940 deletions.
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ go-cve-dictionary
 *.sqlite3-wal
 tags
 /dist/
+integration/diff
diff --git a/GNUmakefile b/GNUmakefile
@@ -86,25 +86,26 @@ clean-integration:
 	-pkill go-cve.old
 	-pkill go-cve.new
 	-rm integration/go-cve.old integration/go-cve.new integration/cve.old.sqlite3 integration/cve.new.sqlite3
+	-rm -rf integration/diff
 	-docker kill redis-old redis-new
 	-docker rm redis-old redis-new
 
 fetch-rdb:
-	integration/go-cve.old fetch nvd --dbpath=$(PWD)/integration/cve.old.sqlite3 --last2y
-	integration/go-cve.old fetch jvn --dbpath=$(PWD)/integration/cve.old.sqlite3 --last2y
+	integration/go-cve.old fetch nvd --dbpath=$(PWD)/integration/cve.old.sqlite3
+	integration/go-cve.old fetch jvn --dbpath=$(PWD)/integration/cve.old.sqlite3
 
-	integration/go-cve.new fetch nvd --dbpath=$(PWD)/integration/cve.new.sqlite3 --last2y
-	integration/go-cve.new fetch jvn --dbpath=$(PWD)/integration/cve.new.sqlite3 --last2y
+	integration/go-cve.new fetch nvd --dbpath=$(PWD)/integration/cve.new.sqlite3
+	integration/go-cve.new fetch jvn --dbpath=$(PWD)/integration/cve.new.sqlite3
 
 fetch-redis:
 	docker run --name redis-old -d -p 127.0.0.1:6379:6379 redis
 	docker run --name redis-new -d -p 127.0.0.1:6380:6379 redis
 
-	integration/go-cve.old fetch nvd --dbtype redis --dbpath "redis://127.0.0.1:6379/0" --last2y
-	integration/go-cve.old fetch jvn --dbtype redis --dbpath "redis://127.0.0.1:6379/0" --last2y
+	integration/go-cve.old fetch nvd --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
+	integration/go-cve.old fetch jvn --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
 
-	integration/go-cve.new fetch nvd --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --last2y
-	integration/go-cve.new fetch jvn --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --last2y
+	integration/go-cve.new fetch nvd --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
+	integration/go-cve.new fetch jvn --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
 
 diff-cveid:
 	@ python integration/diff_server_mode.py cves --sample_rate 0.01

diff --git a/README.md b/README.md
@@ -66,10 +66,8 @@ $ sudo chmod 700 /var/log/go-cve-dictionary
 ```
 
 Fetch vulnerability data from NVD.
-It takes about 10 minutes (on AWS).
-
 ```bash
-$ for i in `seq 2002 $(date +"%Y")`; do go-cve-dictionary fetch nvd --years $i; done
+$ go-cve-dictionary fetch nvd
 ... snip ...
 $ ls -alh cve.sqlite3
 -rw-r--r-- 1 ec2-user ec2-user 7.0M Mar 24 13:20 cve.sqlite3
@@ -215,7 +213,6 @@ Available Commands:
   completion  generate the autocompletion script for the specified shell
   fetch       Fetch Vulnerability dictionary
   help        Help about any command
-  list        Show a list of fetched feeds
   server      Start CVE dictionary HTTP Server
 
 Flags:
@@ -252,11 +249,7 @@ Available Commands:
 
 Flags:
       --batch-size int   The number of batch size to insert. NOTE: This Option does not work for dbtype: redis. (default 5)
-      --force            Force update
   -h, --help             help for fetch
-      --last2y           fetch CVE data in the last two years
-      --latest           fetch CVE data for latest
-      --years            fetch CVE data of specific years
 
 Global Flags:
       --config string       config file (default is $HOME/.go-cve-dictionary.yaml)
@@ -273,55 +266,14 @@ Use "go-cve-dictionary fetch [command] --help" for more information about a comm
 ```
 
 #### Fetch NVD data
-- Fetch data for entire period.
-
-    ```bash
-    for i in `seq 2002 $(date +"%Y")`; do go-cve-dictionary fetch nvd --years $i; done
-    ```
-
-- Fetch data in the last 8 days
-
-    ```bash
-    $ go-cve-dictionary fetch nvd --latest
-    ```
-
-- Fetch data in the last two years
-
-    ```bash
-    $ go-cve-dictionary fetch nvd --last2y
-    ```
-
-- Fetch data of specific years
-
-    ```bash
-    $ go-cve-dictionary fetch nvd --years 2002 2003 2016
-    ```
+```bash
+$ go-cve-dictionary fetch nvd
+```
 
 #### Fetch JVN data
-
-- Fetch data for entire period.
-    ```bash
-    for i in `seq 1998 $(date +"%Y")`; do go-cve-dictionary fetch jvn --years $i; done
-    ```
-
-- Fetch data in the last two years
-
-    ```bash
-    $ go-cve-dictionary fetch jvn --last2y
-    ```
-
-- Fetch data of specific years
-
-    ```bash
-    $ go-cve-dictionary fetch jvn --years 2002 2003 2016
-    ```
-
-- Fetch data for latest
-
-    ```bash
-    $ go-cve-dictionary fetch jvn --latest
-    ```
-
+```bash
+$ go-cve-dictionary fetch jvn
+```
 ----
 
 ### Usage: Run HTTP Server
@@ -357,15 +309,15 @@ Global Flags:
 - fetch nvd
 
     ```bash
-    $ go-cve-dictionary fetch nvd --last2y \
+    $ go-cve-dictionary fetch nvd \
           --dbtype mysql \
           --dbpath "user:pass@tcp(localhost:3306)/dbname?parseTime=true"
     ```
 
 - fetch jvn
 
     ```bash
-    $ go-cve-dictionary fetch jvn --last2y \
+    $ go-cve-dictionary fetch jvn \
           --dbtype mysql \
           --dbpath "user:pass@tcp(localhost:3306)/dbname?parseTime=true"
     ```
@@ -383,15 +335,15 @@ Global Flags:
 - fetch nvd
 
     ```bash
-    $ go-cve-dictionary fetch nvd --last2y \
+    $ go-cve-dictionary fetch nvd \
           --dbtype postgres \
           --dbpath "host=myhost user=user dbname=dbname sslmode=disable password=password"
     ```
 
 - fetch jvn
 
     ```bash
-    $ go-cve-dictionary fetch jvn --last2y \
+    $ go-cve-dictionary fetch jvn \
           --dbtype postgres \
           --dbpath "host=myhost user=user dbname=dbname sslmode=disable password=password"
     ```
@@ -409,15 +361,15 @@ Global Flags:
 - fetch nvd
 
     ```bash
-    $ go-cve-dictionary fetch nvd --last2y \
+    $ go-cve-dictionary fetch nvd \
           --dbtype redis \
           --dbpath "redis://localhost/0"
     ```
 
 - fetch jvn
 
     ```bash
-    $ go-cve-dictionary fetch jvn --last2y \
+    $ go-cve-dictionary fetch jvn \
           --dbtype redis \
           --dbpath "redis://localhost/0"
     ```
@@ -440,9 +392,6 @@ If your system at behind HTTP proxy, you have to specify -http-proxy option.
 - How to daemonize go-cve-dictionary  
 Use Systemd, Upstart or supervisord, daemontools...
 
-- How to update vulnerability data automatically  
-Use job scheduler like cron (with --last2y or --latest option).
-
 - How to cross compile
 
     ```bash

diff --git a/commands/fetch.go b/commands/fetch.go
@@ -14,18 +14,6 @@ var fetchCmd = &cobra.Command{
 func init() {
 	RootCmd.AddCommand(fetchCmd)
 
-	fetchCmd.PersistentFlags().Bool("latest", false, "fetch CVE data for latest")
-	_ = viper.BindPFlag("latest", fetchCmd.PersistentFlags().Lookup("latest"))
-
-	fetchCmd.PersistentFlags().Bool("last2y", false, "fetch CVE data in the last two years")
-	_ = viper.BindPFlag("last2y", fetchCmd.PersistentFlags().Lookup("last2y"))
-
-	fetchCmd.PersistentFlags().Bool("years", false, "fetch CVE data of specific years")
-	_ = viper.BindPFlag("years", fetchCmd.PersistentFlags().Lookup("years"))
-
-	fetchCmd.PersistentFlags().Bool("force", false, "Force update")
-	_ = viper.BindPFlag("force", fetchCmd.PersistentFlags().Lookup("force"))
-
 	fetchCmd.PersistentFlags().Int("batch-size", 5, "The number of batch size to insert. NOTE: This Option does not work for dbtype: redis.")
 	_ = viper.BindPFlag("batch-size", fetchCmd.PersistentFlags().Lookup("batch-size"))
 }
diff --git a/commands/fetchjvn.go b/commands/fetchjvn.go
@@ -1,8 +1,6 @@
 package commands
 
 import (
-	"fmt"
-	"strconv"
 	"time"
 
 	c "github.com/kotakanbe/go-cve-dictionary/config"
@@ -31,56 +29,6 @@ func fetchJvn(cmd *cobra.Command, args []string) (err error) {
 		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
 	}
 
-	years := []int{}
-	thisYear := time.Now().Year()
-
-	switch {
-	case viper.GetBool("latest"):
-		years = append(years, c.Latest)
-	case viper.GetBool("last2y"):
-		for i := 0; i < 2; i++ {
-			years = append(years, thisYear-i)
-		}
-		years = append(years, c.Latest)
-	case viper.GetBool("years"):
-		if len(args) == 0 {
-			errMsg := fmt.Sprintf("Specify years to fetch (from 1998 to %d)", thisYear)
-			log.Errorf(errMsg)
-			return xerrors.Errorf("Failed to fetchJvn. err: %s", errMsg)
-		}
-
-		for _, arg := range args {
-			year, err := strconv.Atoi(arg)
-			if err != nil {
-				return xerrors.Errorf("Failed to strconv.Atoi(%s). err: %w", arg, err)
-			}
-			years = append(years, year)
-		}
-
-		for _, year := range years {
-			if err != nil || year < 1998 || time.Now().Year() < year {
-				errMsg := fmt.Sprintf("Specify years to fetch (from 1998 to %d), arg: %d", thisYear, year)
-				log.Errorf(errMsg)
-				return xerrors.Errorf("Failed to fetchJvn. err: %s", errMsg)
-			}
-			found := false
-			for _, y := range years {
-				if y == year {
-					found = true
-					break
-				}
-			}
-			if !found {
-				years = append(years, year)
-			}
-		}
-		years = append(years, c.Latest)
-	default:
-		errMsg := "specify --latest, --last2y or --years"
-		log.Errorf(errMsg)
-		return xerrors.New(errMsg)
-	}
-
 	driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"))
 	if err != nil {
 		if locked {
@@ -99,48 +47,17 @@ func fetchJvn(cmd *cobra.Command, args []string) (err error) {
 		return xerrors.New("Failed to Insert CVEs into DB. SchemaVersion is old")
 	}
 
-	metas, err := jvn.FetchLatestFeedMeta(driver, years)
-	if err != nil {
-		log.Errorf("%s", err)
-		return err
-	}
-
-	if len(metas) == 0 {
-		log.Errorf("No meta files fetched")
-		return err
-	}
-
-	needUpdates := []models.FeedMeta{}
-	if viper.GetBool("force") {
-		needUpdates = metas
-	} else {
-		for _, m := range metas {
-			if m.Newly() {
-				needUpdates = append(needUpdates, m)
-				log.Infof("Newly     : %s", m.URL)
-			} else if m.OutDated() {
-				needUpdates = append(needUpdates, m)
-				log.Infof("Outdated  : %s", m.URL)
-			} else {
-				log.Infof("Up to date: %s", m.URL)
-			}
-		}
+	years := []int{c.Latest}
+	for y := 1998; y <= time.Now().Year(); y++ {
+		years = append(years, y)
 	}
-
-	if len(needUpdates) == 0 {
-		log.Infof("Already up to date")
-		return nil
-	}
-
-	log.Infof("Fetching CVE information from JVN.")
-	cves, err := jvn.FetchConvert(needUpdates)
+	feedMetas, err := jvn.FetchLatestFeedMeta(years)
 	if err != nil {
-		log.Errorf("Failed to fetch JVN: %s", err)
 		return err
 	}
 
 	log.Infof("Inserting JVN into DB (%s).", driver.Name())
-	if err := driver.InsertJvn(needUpdates, cves); err != nil {
+	if err := driver.InsertJvn(feedMetas); err != nil {
 		log.Fatalf("Failed to insert. dbpath: %s, err: %s", viper.GetString("dbpath"), err)
 		return err
 	}