feat: include and exclude rules (#113)

- fix: don't try to scan util commands
- feat: print all rules with their tags
- ci: check needed go mod tidy
- chore: find duplicate rules
- feat: include and exclude rules

Closes #28

---------

Co-authored-by: Jossef Harush Kadouri <jossef12@gmail.com>

README.md

@@ -18,11 +18,11 @@ During the software development lifecycle (SDLC), developers ofen communicate an
 - Git
 - Paligo
 - Local directory / files
-  
+
 ## Getting 2ms
 
 ```
-# go install github.com/checkmarx/2ms@latest
+go install github.com/checkmarx/2ms@latest
 ```
 
 ### Docker

cmd/main.go

@@ -28,19 +28,21 @@ const (
 	yamlFormat        = "yaml"
 	sarifFormat       = "sarif"
 
-	tagsFlagName            = "tags"
 	logLevelFlagName        = "log-level"
 	reportPathFlagName      = "report-path"
 	stdoutFormatFlagName    = "stdout-format"
 	customRegexRuleFlagName = "regex"
+	includeRuleFlagName     = "include-rule"
+	excludeRuleFlagName     = "exclude-rule"
 )
 
 var (
-	tagsVar            []string
 	logLevelVar        string
 	reportPathVar      []string
 	stdoutFormatVar    string
 	customRegexRuleVar []string
+	includeRuleVar     []string
+	excludeRuleVar     []string
 )
 
 var rootCmd = &cobra.Command{
@@ -89,24 +91,28 @@ func initLog() {
 
 func Execute() {
 	cobra.OnInitialize(initLog)
-	rootCmd.PersistentFlags().StringSliceVar(&tagsVar, tagsFlagName, []string{"all"}, "select rules to be applied")
 	rootCmd.PersistentFlags().StringVar(&logLevelVar, logLevelFlagName, "info", "log level (trace, debug, info, warn, error, fatal)")
 	rootCmd.PersistentFlags().StringSliceVar(&reportPathVar, reportPathFlagName, []string{}, "path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)")
 	rootCmd.PersistentFlags().StringVar(&stdoutFormatVar, stdoutFormatFlagName, "yaml", "stdout output format, available formats are: json, yaml, sarif")
 	rootCmd.PersistentFlags().StringArrayVar(&customRegexRuleVar, customRegexRuleFlagName, []string{}, "custom regexes to apply to the scan, must be valid Go regex")
 
-	rootCmd.PersistentPreRun = preRun
-	rootCmd.PersistentPostRun = postRun
+	rootCmd.PersistentFlags().StringSliceVar(&includeRuleVar, includeRuleFlagName, []string{}, "include rules by name or tag to apply to the scan (adds to list, starts from empty)")
+	rootCmd.PersistentFlags().StringSliceVar(&excludeRuleVar, excludeRuleFlagName, []string{}, "exclude rules by name or tag to apply to the scan (removes from list, starts from all)")
+	rootCmd.MarkFlagsMutuallyExclusive(includeRuleFlagName, excludeRuleFlagName)
+
+	rootCmd.AddCommand(secrets.RulesCommand)
 
 	group := "Commands"
 	rootCmd.AddGroup(&cobra.Group{Title: group, ID: group})
 
 	for _, plugin := range allPlugins {
 		subCommand, err := plugin.DefineCommand(channels)
-		subCommand.GroupID = group
 		if err != nil {
 			log.Fatal().Msg(fmt.Sprintf("error while defining command for plugin %s: %s", plugin.GetName(), err.Error()))
 		}
+		subCommand.GroupID = group
+		subCommand.PreRun = preRun
+		subCommand.PostRun = postRun
 		rootCmd.AddCommand(subCommand)
 	}
 
@@ -115,20 +121,6 @@ func Execute() {
 	}
 }
 
-func validateTags(tags []string) {
-	for _, tag := range tags {
-		if !(strings.EqualFold(tag, "all") || strings.EqualFold(tag, secrets.TagApiKey) || strings.EqualFold(tag, secrets.TagClientId) ||
-			strings.EqualFold(tag, secrets.TagClientSecret) || strings.EqualFold(tag, secrets.TagSecretKey) || strings.EqualFold(tag, secrets.TagAccessKey) ||
-			strings.EqualFold(tag, secrets.TagAccessId) || strings.EqualFold(tag, secrets.TagApiToken) || strings.EqualFold(tag, secrets.TagAccessToken) ||
-			strings.EqualFold(tag, secrets.TagRefreshToken) || strings.EqualFold(tag, secrets.TagPrivateKey) || strings.EqualFold(tag, secrets.TagPublicKey) ||
-			strings.EqualFold(tag, secrets.TagEncryptionKey) || strings.EqualFold(tag, secrets.TagTriggerToken) || strings.EqualFold(tag, secrets.TagRegistrationToken) ||
-			strings.EqualFold(tag, secrets.TagPassword) || strings.EqualFold(tag, secrets.TagUploadToken) || strings.EqualFold(tag, secrets.TagPublicSecret) ||
-			strings.EqualFold(tag, secrets.TagSensitiveUrl) || strings.EqualFold(tag, secrets.TagWebhook)) {
-			log.Fatal().Msgf(`invalid filter: %s`, tag)
-		}
-	}
-}
-
 func validateFormat(stdout string, reportPath []string) {
 	if !(strings.EqualFold(stdout, yamlFormat) || strings.EqualFold(stdout, jsonFormat) || strings.EqualFold(stdout, sarifFormat)) {
 		log.Fatal().Msgf(`invalid output format: %s, available formats are: json, yaml and sarif`, stdout)
@@ -144,9 +136,10 @@ func validateFormat(stdout string, reportPath []string) {
 }
 
 func preRun(cmd *cobra.Command, args []string) {
-	validateTags(tagsVar)
-
-	secrets := secrets.Init(tagsVar)
+	secrets, err := secrets.Init(includeRuleVar, excludeRuleVar)
+	if err != nil {
+		log.Fatal().Msg(err.Error())
+	}
 
 	if err := secrets.AddRegexRules(customRegexRuleVar); err != nil {
 		log.Fatal().Msg(err.Error())

secrets/secrets.go

@@ -2,13 +2,16 @@ package secrets
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 	"sync"
+	"text/tabwriter"
 
 	"github.com/checkmarx/2ms/plugins"
 	"github.com/checkmarx/2ms/reporting"
+	"github.com/spf13/cobra"
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules"
 	"github.com/zricethezav/gitleaks/v8/config"
 	"github.com/zricethezav/gitleaks/v8/detect"
@@ -46,10 +49,27 @@ const TagWebhook = "webhook"
 
 const customRegexRuleIdFormat = "custom-regex-%d"
 
-func Init(tags []string) *Secrets {
+func Init(includeList, excludeList []string) (*Secrets, error) {
+	if len(includeList) > 0 && len(excludeList) > 0 {
+		return nil, fmt.Errorf("cannot use both include and exclude flags")
+	}
 
 	allRules, _ := loadAllRules()
-	rulesToBeApplied := getRules(allRules, tags)
+	rulesToBeApplied := make(map[string]config.Rule)
+	if len(includeList) > 0 {
+		rulesToBeApplied = selectRules(allRules, includeList)
+	} else if len(excludeList) > 0 {
+		rulesToBeApplied = excludeRules(allRules, excludeList)
+	} else {
+		for _, rule := range allRules {
+			// required to be empty when not running via cli. otherwise rule will be ignored
+			rule.Rule.Keywords = []string{}
+			rulesToBeApplied[rule.Rule.RuleID] = rule.Rule
+		}
+	}
+	if len(rulesToBeApplied) == 0 {
+		return nil, fmt.Errorf("no rules were selected")
+	}
 
 	config := config.Config{
 		Rules: rulesToBeApplied,
@@ -60,7 +80,7 @@ func Init(tags []string) *Secrets {
 	return &Secrets{
 		rules:    rulesToBeApplied,
 		detector: *detector,
-	}
+	}, nil
 }
 
 func (s *Secrets) Detect(secretsChannel chan reporting.Secret, item plugins.Item, wg *sync.WaitGroup) {
@@ -104,6 +124,46 @@ func getItemId(fullPath string) string {
 	return itemId
 }
 
+func selectRules(allRules []Rule, tags []string) map[string]config.Rule {
+	rulesToBeApplied := make(map[string]config.Rule)
+
+	for _, rule := range allRules {
+		if isRuleMatch(rule, tags) {
+			// required to be empty when not running via cli. otherwise rule will be ignored
+			rule.Rule.Keywords = []string{}
+			rulesToBeApplied[rule.Rule.RuleID] = rule.Rule
+		}
+	}
+	return rulesToBeApplied
+}
+
+func excludeRules(allRules []Rule, tags []string) map[string]config.Rule {
+	rulesToBeApplied := make(map[string]config.Rule)
+
+	for _, rule := range allRules {
+		if !isRuleMatch(rule, tags) {
+			// required to be empty when not running via cli. otherwise rule will be ignored
+			rule.Rule.Keywords = []string{}
+			rulesToBeApplied[rule.Rule.RuleID] = rule.Rule
+		}
+	}
+	return rulesToBeApplied
+}
+
+func isRuleMatch(rule Rule, tags []string) bool {
+	for _, tag := range tags {
+		if strings.EqualFold(rule.Rule.RuleID, tag) {
+			return true
+		}
+		for _, ruleTag := range rule.Tags {
+			if strings.EqualFold(ruleTag, tag) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func getRules(allRules []Rule, tags []string) map[string]config.Rule {
 	rulesToBeApplied := make(map[string]config.Rule)
 
@@ -166,7 +226,7 @@ func loadAllRules() ([]Rule, error) {
 	allRules = append(allRules, Rule{Rule: *rules.ConfluentSecretKey(), Tags: []string{TagSecretKey}})
 	allRules = append(allRules, Rule{Rule: *rules.Contentful(), Tags: []string{TagApiToken}})
 	allRules = append(allRules, Rule{Rule: *rules.Databricks(), Tags: []string{TagApiToken}})
-	allRules = append(allRules, Rule{Rule: *rules.DatadogtokenAccessToken(), Tags: []string{TagAccessToken}})
+	allRules = append(allRules, Rule{Rule: *rules.DatadogtokenAccessToken(), Tags: []string{TagAccessToken, TagClientId}})
 	allRules = append(allRules, Rule{Rule: *rules.DigitalOceanPAT(), Tags: []string{TagAccessToken}})
 	allRules = append(allRules, Rule{Rule: *rules.DigitalOceanOAuthToken(), Tags: []string{TagAccessToken}})
 	allRules = append(allRules, Rule{Rule: *rules.DigitalOceanRefreshToken(), Tags: []string{TagRefreshToken}})
@@ -178,7 +238,6 @@ func loadAllRules() ([]Rule, error) {
 	allRules = append(allRules, Rule{Rule: *rules.DropBoxShortLivedAPIToken(), Tags: []string{TagApiToken}})
 	allRules = append(allRules, Rule{Rule: *rules.DropBoxLongLivedAPIToken(), Tags: []string{TagApiToken}})
 	allRules = append(allRules, Rule{Rule: *rules.DroneciAccessToken(), Tags: []string{TagAccessToken}})
-	allRules = append(allRules, Rule{Rule: *rules.DatadogtokenAccessToken(), Tags: []string{TagClientId}})
 	allRules = append(allRules, Rule{Rule: *rules.Duffel(), Tags: []string{TagApiToken}})
 	allRules = append(allRules, Rule{Rule: *rules.Dynatrace(), Tags: []string{TagApiToken}})
 	allRules = append(allRules, Rule{Rule: *rules.EasyPost(), Tags: []string{TagApiToken}})
@@ -293,3 +352,29 @@ func loadAllRules() ([]Rule, error) {
 
 	return allRules, nil
 }
+
+var RulesCommand = &cobra.Command{
+	Use:   "rules",
+	Short: "List all rules",
+	Long:  `List all rules`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+
+		rules, err := loadAllRules()
+		if err != nil {
+			return err
+		}
+
+		tab := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
+
+		fmt.Fprintln(tab, "Name\tDescription\tTags")
+		fmt.Fprintln(tab, "----\t----\t----")
+		for _, rule := range rules {
+			fmt.Fprintf(tab, "%s\t%s\t%s\n", rule.Rule.RuleID, rule.Rule.Description, strings.Join(rule.Tags, ","))
+		}
+		if err = tab.Flush(); err != nil {
+			return err
+		}
+
+		return nil
+	},
+}