feat(contrib/fvuls) Add commands to obtained CPE information of network devices by executing snmp2cpe and upload to Fvuls server

.gitignore

@@ -3,22 +3,22 @@
 *.swp
 *.sqlite3*
 *.db
+*.toml
 tags
 .gitmodules
 coverage.out
 issues/
 vendor/
 log/
 results
-config.toml
 !setup/docker/*
 .DS_Store
 dist/
 .idea
 vuls.*
 vuls
 !cmd/vuls
-future-vuls
-trivy-to-vuls
+/future-vuls
+/trivy-to-vuls
 snmp2cpe
-!snmp2cpe/
+!snmp2cpe/

contrib/future-vuls/README.md

@@ -2,18 +2,77 @@
 
 ## Main Features
 
-- upload vuls results json to future-vuls
+- `future-vuls upload` 
+  - upload vuls results json to future-vuls
 
+- `future-vuls discover`
+ -  Explore hosts within the CIDR range using the ping command
+ -  Describe the information including CPE on the found hosts in a toml-formatted file.
+ -  Exec snmp2cpe(https://github.com/future-architect/vuls/pull/1625) to active hosts to obtain CPE<br>
+Commands running internally　　`snmp2cpe v2c {IPAddr} public  | snmp2cpe convert`<br>
+
+Structure of toml-formatted file
+```
+[server.{ip}]
+ip = {IpAddr}
+server_name = ""
+uuid = {UUID}
+cpe_uris = []
+fvuls_sync = false
+```
+
+- `future-vuls add-cpe`
+  -  Create pseudo server to Fvuls to obtain uuid and Upload CPE information on the specified(FvulsSync is true and UUID is obtained) hosts to Fvuls
+  -  Fvuls_Sync must be rewritten to true to designate it as the target of the command<br><br>
+
+
+1.　`future-vuls discover`
+
+2.　`future-vuls add-cpe`
+
+These two commands are used to manage the CPE of network devices, and by executing the commands in the order from the top, you can manage the CPE of each device in Fvuls
+
+toml file after command execution
+```
+["192.168.0.10"]
+  ip = "192.168.0.10"
+  server_name = "192.168.0.10"
+  uuid = "e811e2b1-9463-d682-7c79-a4ab37de28cf"
+  cpe_uris = ["cpe:2.3:h:fortinet:fortigate-50e:-:*:*:*:*:*:*:*", "cpe:2.3:o:fortinet:fortios:5.4.6:*:*:*:*:*:*:*"]
+  fvuls_sync = true
+```
 ## Installation
 
 ```
 git clone https://github.com/future-architect/vuls.git
+cd vuls
 make build-future-vuls
 ```
 
 ## Command Reference
 
 ```
+./future-vuls -h
+Usage:
+  future-vuls [command]
+
+Available Commands:
+  add-cpe     Create a pseudo server in Fvuls and register CPE. Default outputFile is ./discover_list.toml
+  completion  Generate the autocompletion script for the specified shell
+  discover    discover hosts with CIDR range. Run snmp2cpe on active host to get CPE. Default outputFile is ./discover_list.toml
+  help        Help about any command
+  upload      Upload to FutureVuls
+  version     Show version
+
+Flags:
+  -h, --help   help for future-vuls
+
+Use "future-vuls [command] --help" for more information about a command.
+```
+### Subcommands
+
+```
+./future-vuls upload -h
 Upload to FutureVuls
 
 Usage:
@@ -29,10 +88,71 @@ Flags:
       --uuid string     server uuid. ENV: VULS_SERVER_UUID
 ```
 
+```
+./future-vuls discover -h
+discover hosts with CIDR range. Run snmp2cpe on active host to get CPE. Default outputFile is ./discover_list.toml
+
+Usage:
+  future-vuls discover --cidr <CIDR_RANGE> --output <OUTPUT_FILE> [flags]
+
+Examples:
+future-vuls discover --cidr 192.168.0.0/24 --output discover_list.toml
+
+Flags:
+      --cidr string           cidr range
+  -h, --help                  help for discover
+      --output string         output file
+      --snmp-version string   snmp version v1,v2c and v3. default: v2c
+```
+
+```
+./future-vuls add-cpe -h
+Create a pseudo server in Fvuls and register CPE. Default outputFile is ./discover_list.toml
+
+Usage:
+  future-vuls add-cpe --token <VULS_TOKEN> --output <OUTPUT_FILE> [flags]
+
+Examples:
+future-vuls add-cpe --token <VULS_TOKEN>
+
+Flags:
+  -h, --help                help for add-cpe
+      --http-proxy string   proxy url
+      --output string       output file
+  -t, --token string        future vuls token ENV: VULS_TOKEN
+```
+
 ## Usage
 
-- update results json
+- `future-vuls upload`
 
 ```
  cat results.json | future-vuls upload --stdin --token xxxx --url https://xxxx --group-id 1 --uuid xxxx
-```
+```
+- `future-vuls discover`
+```
+./future-vuls discover --cidr 192.168.0.1/24
+Discovering 192.168.0.1/24...
+192.168.0.1: Execute snmp2cpe...
+failed to execute snmp2cpe. err: failed to execute snmp2cpe. err: exit status 1
+192.168.0.2: Execute snmp2cpe...
+failed to execute snmp2cpe. err: failed to execute snmp2cpe. err: exit status 1
+192.168.0.4: Execute snmp2cpe...
+failed to execute snmp2cpe. err: failed to execute snmp2cpe. err: exit status 1
+192.168.0.5: Execute snmp2cpe...
+failed to execute snmp2cpe. err: failed to execute snmp2cpe. err: exit status 1
+192.168.0.6: Execute snmp2cpe...
+New network device found 192.168.0.6
+wrote to discover_list.toml
+```
+- `future-vuls add-cpe`
+```
+./future-vuls add-cpe --token fvgr-686b92af-5216-11ee-a241-0a58a9feac02
+Creating 1 pseudo server...
+192.168.0.6: Created FutureVuls pseudo server ce024b45-1c59-5b86-1a67-e78a40dfec01
+wrote to discover_list.toml
+
+Uploading 1 server's CPE...
+192.168.0.6: Uploaded CPE cpe:2.3:h:fortinet:fortigate-50e:-:*:*:*:*:*:*:*
+192.168.0.6: Uploaded CPE cpe:2.3:o:fortinet:fortios:5.4.6:*:*:*:*:*:*:*
+```

contrib/future-vuls/cmd/main.go

@@ -1,118 +1,162 @@
+// Package main ...
 package main
 
 import (
 	"bufio"
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"os"
 	"strconv"
 	"strings"
 
-	"github.com/future-architect/vuls/config"
-	"github.com/future-architect/vuls/models"
-	"github.com/future-architect/vuls/saas"
+	cidrPkg "github.com/3th1nk/cidr"
+	vulsConfig "github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/contrib/future-vuls/pkg/config"
+	"github.com/future-architect/vuls/contrib/future-vuls/pkg/cpe"
+	"github.com/future-architect/vuls/contrib/future-vuls/pkg/discover"
+	"github.com/future-architect/vuls/contrib/future-vuls/pkg/fvuls"
+
 	"github.com/spf13/cobra"
 )
 
 var (
-	configFile string
-	stdIn      bool
-	jsonDir    string
-	serverUUID string
-	groupID    int64
-	token      string
-	tags       []string
-	url        string
+	configFile  string
+	stdIn       bool
+	jsonDir     string
+	serverUUID  string
+	groupID     int64
+	token       string
+	tags        []string
+	outputFile  string
+	cidr        string
+	snmpVersion string
+	proxy       string
 )
 
 func main() {
 	var err error
+	var cmdVersion = &cobra.Command{
+		Use:   "version",
+		Short: "Show version",
+		Long:  "Show version",
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Printf("future-vuls-%s-%s\n", vulsConfig.Version, vulsConfig.Revision)
+		},
+	}
+
 	var cmdFvulsUploader = &cobra.Command{
 		Use:   "upload",
 		Short: "Upload to FutureVuls",
 		Long:  `Upload to FutureVuls`,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if len(serverUUID) == 0 {
 				serverUUID = os.Getenv("VULS_SERVER_UUID")
 			}
 			if groupID == 0 {
 				envGroupID := os.Getenv("VULS_GROUP_ID")
 				if groupID, err = strconv.ParseInt(envGroupID, 10, 64); err != nil {
-					fmt.Printf("Invalid GroupID: %s\n", envGroupID)
-					return
+					return fmt.Errorf("invalid GroupID: %s", envGroupID)
 				}
 			}
-			if len(url) == 0 {
-				url = os.Getenv("VULS_URL")
-			}
 			if len(token) == 0 {
 				token = os.Getenv("VULS_TOKEN")
 			}
 			if len(tags) == 0 {
 				tags = strings.Split(os.Getenv("VULS_TAGS"), ",")
 			}
-
 			var scanResultJSON []byte
 			if stdIn {
 				reader := bufio.NewReader(os.Stdin)
 				buf := new(bytes.Buffer)
-				if _, err = buf.ReadFrom(reader); err != nil {
-					return
+				if _, err := buf.ReadFrom(reader); err != nil {
+					return fmt.Errorf("failed to read from stdIn. err: %v", err)
 				}
 				scanResultJSON = buf.Bytes()
 			} else {
-				fmt.Println("use --stdin option")
+				return fmt.Errorf("use --stdin option")
+			}
+			fvulsClient := fvuls.NewClient(token, "")
+			if err := fvulsClient.UploadToFvuls(serverUUID, groupID, tags, scanResultJSON); err != nil {
+				fmt.Printf("%v", err)
+				// avoid to display help message
 				os.Exit(1)
-				return
 			}
+			return nil
+		},
+	}
 
-			var scanResult models.ScanResult
-			if err = json.Unmarshal(scanResultJSON, &scanResult); err != nil {
-				fmt.Println("Failed to parse json", err)
-				os.Exit(1)
-				return
+	var cmdDiscover = &cobra.Command{
+		Use:     "discover --cidr <CIDR_RANGE> --output <OUTPUT_FILE>",
+		Short:   "discover hosts with CIDR range. Run snmp2cpe on active host to get CPE. Default outputFile is ./discover_list.toml",
+		Example: "future-vuls discover --cidr 192.168.0.0/24 --output discover_list.toml",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(outputFile) == 0 {
+				outputFile = config.DiscoverTomlFileName
 			}
-			scanResult.ServerUUID = serverUUID
-			if 0 < len(tags) {
-				if scanResult.Optional == nil {
-					scanResult.Optional = map[string]interface{}{}
-				}
-				scanResult.Optional["VULS_TAGS"] = tags
+			if len(cidr) == 0 {
+				return fmt.Errorf("please specify cidr range")
 			}
-
-			config.Conf.Saas.GroupID = groupID
-			config.Conf.Saas.Token = token
-			config.Conf.Saas.URL = url
-			if err = (saas.Writer{}).Write(scanResult); err != nil {
-				fmt.Println(err)
+			if _, err := cidrPkg.Parse(cidr); err != nil {
+				return fmt.Errorf("Invalid cidr range")
+			}
+			if len(snmpVersion) == 0 {
+				snmpVersion = config.SnmpVersion
+			}
+			if snmpVersion != "v1" && snmpVersion != "v2c" && snmpVersion != "v3" {
+				return fmt.Errorf("Invalid snmpVersion")
+			}
+			if err := discover.ActiveHosts(cidr, outputFile, snmpVersion); err != nil {
+				fmt.Printf("%v", err)
+				// avoid to display help message
 				os.Exit(1)
-				return
 			}
-			return
+			return nil
 		},
 	}
-	var cmdVersion = &cobra.Command{
-		Use:   "version",
-		Short: "Show version",
-		Long:  "Show version",
-		Run: func(cmd *cobra.Command, args []string) {
-			fmt.Printf("future-vuls-%s-%s\n", config.Version, config.Revision)
+
+	var cmdAddCpe = &cobra.Command{
+		Use:     "add-cpe --token <VULS_TOKEN> --output <OUTPUT_FILE>",
+		Short:   "Create a pseudo server in Fvuls and register CPE. Default outputFile is ./discover_list.toml",
+		Example: "future-vuls add-cpe --token <VULS_TOKEN>",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(token) == 0 {
+				token = os.Getenv("VULS_TOKEN")
+				if len(token) == 0 {
+					return fmt.Errorf("token not specified")
+				}
+			}
+			if len(outputFile) == 0 {
+				outputFile = config.DiscoverTomlFileName
+			}
+			if err := cpe.AddCpe(token, outputFile, proxy); err != nil {
+				fmt.Printf("%v", err)
+				// avoid to display help message
+				os.Exit(1)
+			}
+			return nil
 		},
 	}
+
 	cmdFvulsUploader.PersistentFlags().StringVar(&serverUUID, "uuid", "", "server uuid. ENV: VULS_SERVER_UUID")
 	cmdFvulsUploader.PersistentFlags().StringVar(&configFile, "config", "", "config file (default is $HOME/.cobra.yaml)")
 	cmdFvulsUploader.PersistentFlags().BoolVarP(&stdIn, "stdin", "s", false, "input from stdin. ENV: VULS_STDIN")
-	// TODO Read JSON file from directory
-	//	cmdFvulsUploader.Flags().StringVarP(&jsonDir, "results-dir", "d", "./", "vuls scan results json dir")
 	cmdFvulsUploader.PersistentFlags().Int64VarP(&groupID, "group-id", "g", 0, "future vuls group id, ENV: VULS_GROUP_ID")
 	cmdFvulsUploader.PersistentFlags().StringVarP(&token, "token", "t", "", "future vuls token")
-	cmdFvulsUploader.PersistentFlags().StringVar(&url, "url", "", "future vuls upload url")
+
+	cmdDiscover.PersistentFlags().StringVar(&cidr, "cidr", "", "cidr range")
+	cmdDiscover.PersistentFlags().StringVar(&outputFile, "output", "", "output file")
+	cmdDiscover.PersistentFlags().StringVar(&snmpVersion, "snmp-version", "", "snmp version v1,v2c and v3. default: v2c ")
+
+	cmdAddCpe.PersistentFlags().StringVarP(&token, "token", "t", "", "future vuls token ENV: VULS_TOKEN")
+	cmdAddCpe.PersistentFlags().StringVar(&outputFile, "output", "", "output file")
+	cmdAddCpe.PersistentFlags().StringVar(&proxy, "http-proxy", "", "proxy url")
 
 	var rootCmd = &cobra.Command{Use: "future-vuls"}
+	rootCmd.AddCommand(cmdDiscover)
+	rootCmd.AddCommand(cmdAddCpe)
 	rootCmd.AddCommand(cmdFvulsUploader)
 	rootCmd.AddCommand(cmdVersion)
 	if err = rootCmd.Execute(); err != nil {
-		fmt.Println("Failed to execute command", err)
+		fmt.Println("Failed to execute command")
 	}
 }