diff --git a/README.md b/README.md index 6f4d7d98..41d536dd 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,11 @@ During the software development lifecycle (SDLC), developers ofen communicate an - Git - Paligo - Local directory / files - + ## Getting 2ms ``` -# go install github.com/checkmarx/2ms@latest +go install github.com/checkmarx/2ms@latest ``` ### Docker diff --git a/cmd/main.go b/cmd/main.go index 501b78c5..4d9adeb3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -28,19 +28,21 @@ const ( yamlFormat = "yaml" sarifFormat = "sarif" - tagsFlagName = "tags" logLevelFlagName = "log-level" reportPathFlagName = "report-path" stdoutFormatFlagName = "stdout-format" customRegexRuleFlagName = "regex" + includeRuleFlagName = "include-rule" + excludeRuleFlagName = "exclude-rule" ) var ( - tagsVar []string logLevelVar string reportPathVar []string stdoutFormatVar string customRegexRuleVar []string + includeRuleVar []string + excludeRuleVar []string ) var rootCmd = &cobra.Command{ @@ -89,24 +91,28 @@ func initLog() { func Execute() { cobra.OnInitialize(initLog) - rootCmd.PersistentFlags().StringSliceVar(&tagsVar, tagsFlagName, []string{"all"}, "select rules to be applied") rootCmd.PersistentFlags().StringVar(&logLevelVar, logLevelFlagName, "info", "log level (trace, debug, info, warn, error, fatal)") rootCmd.PersistentFlags().StringSliceVar(&reportPathVar, reportPathFlagName, []string{}, "path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)") rootCmd.PersistentFlags().StringVar(&stdoutFormatVar, stdoutFormatFlagName, "yaml", "stdout output format, available formats are: json, yaml, sarif") rootCmd.PersistentFlags().StringArrayVar(&customRegexRuleVar, customRegexRuleFlagName, []string{}, "custom regexes to apply to the scan, must be valid Go regex") - rootCmd.PersistentPreRun = preRun - rootCmd.PersistentPostRun = postRun + rootCmd.PersistentFlags().StringSliceVar(&includeRuleVar, includeRuleFlagName, []string{}, "include rules by name or tag to apply to the scan (adds to list, starts from empty)") + rootCmd.PersistentFlags().StringSliceVar(&excludeRuleVar, excludeRuleFlagName, []string{}, "exclude rules by name or tag to apply to the scan (removes from list, starts from all)") + rootCmd.MarkFlagsMutuallyExclusive(includeRuleFlagName, excludeRuleFlagName) + + rootCmd.AddCommand(secrets.RulesCommand) group := "Commands" rootCmd.AddGroup(&cobra.Group{Title: group, ID: group}) for _, plugin := range allPlugins { subCommand, err := plugin.DefineCommand(channels) - subCommand.GroupID = group if err != nil { log.Fatal().Msg(fmt.Sprintf("error while defining command for plugin %s: %s", plugin.GetName(), err.Error())) } + subCommand.GroupID = group + subCommand.PreRun = preRun + subCommand.PostRun = postRun rootCmd.AddCommand(subCommand) } @@ -115,20 +121,6 @@ func Execute() { } } -func validateTags(tags []string) { - for _, tag := range tags { - if !(strings.EqualFold(tag, "all") || strings.EqualFold(tag, secrets.TagApiKey) || strings.EqualFold(tag, secrets.TagClientId) || - strings.EqualFold(tag, secrets.TagClientSecret) || strings.EqualFold(tag, secrets.TagSecretKey) || strings.EqualFold(tag, secrets.TagAccessKey) || - strings.EqualFold(tag, secrets.TagAccessId) || strings.EqualFold(tag, secrets.TagApiToken) || strings.EqualFold(tag, secrets.TagAccessToken) || - strings.EqualFold(tag, secrets.TagRefreshToken) || strings.EqualFold(tag, secrets.TagPrivateKey) || strings.EqualFold(tag, secrets.TagPublicKey) || - strings.EqualFold(tag, secrets.TagEncryptionKey) || strings.EqualFold(tag, secrets.TagTriggerToken) || strings.EqualFold(tag, secrets.TagRegistrationToken) || - strings.EqualFold(tag, secrets.TagPassword) || strings.EqualFold(tag, secrets.TagUploadToken) || strings.EqualFold(tag, secrets.TagPublicSecret) || - strings.EqualFold(tag, secrets.TagSensitiveUrl) || strings.EqualFold(tag, secrets.TagWebhook)) { - log.Fatal().Msgf(`invalid filter: %s`, tag) - } - } -} - func validateFormat(stdout string, reportPath []string) { if !(strings.EqualFold(stdout, yamlFormat) || strings.EqualFold(stdout, jsonFormat) || strings.EqualFold(stdout, sarifFormat)) { log.Fatal().Msgf(`invalid output format: %s, available formats are: json, yaml and sarif`, stdout) @@ -144,9 +136,10 @@ func validateFormat(stdout string, reportPath []string) { } func preRun(cmd *cobra.Command, args []string) { - validateTags(tagsVar) - - secrets := secrets.Init(tagsVar) + secrets, err := secrets.Init(includeRuleVar, excludeRuleVar) + if err != nil { + log.Fatal().Msg(err.Error()) + } if err := secrets.AddRegexRules(customRegexRuleVar); err != nil { log.Fatal().Msg(err.Error()) diff --git a/secrets/secrets.go b/secrets/secrets.go index 6855467d..399fa57f 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -2,13 +2,16 @@ package secrets import ( "fmt" + "os" "path/filepath" "regexp" "strings" "sync" + "text/tabwriter" "github.com/checkmarx/2ms/plugins" "github.com/checkmarx/2ms/reporting" + "github.com/spf13/cobra" "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" "github.com/zricethezav/gitleaks/v8/config" "github.com/zricethezav/gitleaks/v8/detect" @@ -46,10 +49,27 @@ const TagWebhook = "webhook" const customRegexRuleIdFormat = "custom-regex-%d" -func Init(tags []string) *Secrets { +func Init(includeList, excludeList []string) (*Secrets, error) { + if len(includeList) > 0 && len(excludeList) > 0 { + return nil, fmt.Errorf("cannot use both include and exclude flags") + } allRules, _ := loadAllRules() - rulesToBeApplied := getRules(allRules, tags) + rulesToBeApplied := make(map[string]config.Rule) + if len(includeList) > 0 { + rulesToBeApplied = selectRules(allRules, includeList) + } else if len(excludeList) > 0 { + rulesToBeApplied = excludeRules(allRules, excludeList) + } else { + for _, rule := range allRules { + // required to be empty when not running via cli. otherwise rule will be ignored + rule.Rule.Keywords = []string{} + rulesToBeApplied[rule.Rule.RuleID] = rule.Rule + } + } + if len(rulesToBeApplied) == 0 { + return nil, fmt.Errorf("no rules were selected") + } config := config.Config{ Rules: rulesToBeApplied, @@ -60,7 +80,7 @@ func Init(tags []string) *Secrets { return &Secrets{ rules: rulesToBeApplied, detector: *detector, - } + }, nil } func (s *Secrets) Detect(secretsChannel chan reporting.Secret, item plugins.Item, wg *sync.WaitGroup) { @@ -104,6 +124,46 @@ func getItemId(fullPath string) string { return itemId } +func selectRules(allRules []Rule, tags []string) map[string]config.Rule { + rulesToBeApplied := make(map[string]config.Rule) + + for _, rule := range allRules { + if isRuleMatch(rule, tags) { + // required to be empty when not running via cli. otherwise rule will be ignored + rule.Rule.Keywords = []string{} + rulesToBeApplied[rule.Rule.RuleID] = rule.Rule + } + } + return rulesToBeApplied +} + +func excludeRules(allRules []Rule, tags []string) map[string]config.Rule { + rulesToBeApplied := make(map[string]config.Rule) + + for _, rule := range allRules { + if !isRuleMatch(rule, tags) { + // required to be empty when not running via cli. otherwise rule will be ignored + rule.Rule.Keywords = []string{} + rulesToBeApplied[rule.Rule.RuleID] = rule.Rule + } + } + return rulesToBeApplied +} + +func isRuleMatch(rule Rule, tags []string) bool { + for _, tag := range tags { + if strings.EqualFold(rule.Rule.RuleID, tag) { + return true + } + for _, ruleTag := range rule.Tags { + if strings.EqualFold(ruleTag, tag) { + return true + } + } + } + return false +} + func getRules(allRules []Rule, tags []string) map[string]config.Rule { rulesToBeApplied := make(map[string]config.Rule) @@ -166,7 +226,7 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.ConfluentSecretKey(), Tags: []string{TagSecretKey}}) allRules = append(allRules, Rule{Rule: *rules.Contentful(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.Databricks(), Tags: []string{TagApiToken}}) - allRules = append(allRules, Rule{Rule: *rules.DatadogtokenAccessToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.DatadogtokenAccessToken(), Tags: []string{TagAccessToken, TagClientId}}) allRules = append(allRules, Rule{Rule: *rules.DigitalOceanPAT(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.DigitalOceanOAuthToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.DigitalOceanRefreshToken(), Tags: []string{TagRefreshToken}}) @@ -178,7 +238,6 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.DropBoxShortLivedAPIToken(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.DropBoxLongLivedAPIToken(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.DroneciAccessToken(), Tags: []string{TagAccessToken}}) - allRules = append(allRules, Rule{Rule: *rules.DatadogtokenAccessToken(), Tags: []string{TagClientId}}) allRules = append(allRules, Rule{Rule: *rules.Duffel(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.Dynatrace(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.EasyPost(), Tags: []string{TagApiToken}}) @@ -293,3 +352,29 @@ func loadAllRules() ([]Rule, error) { return allRules, nil } + +var RulesCommand = &cobra.Command{ + Use: "rules", + Short: "List all rules", + Long: `List all rules`, + RunE: func(cmd *cobra.Command, args []string) error { + + rules, err := loadAllRules() + if err != nil { + return err + } + + tab := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0) + + fmt.Fprintln(tab, "Name\tDescription\tTags") + fmt.Fprintln(tab, "----\t----\t----") + for _, rule := range rules { + fmt.Fprintf(tab, "%s\t%s\t%s\n", rule.Rule.RuleID, rule.Rule.Description, strings.Join(rule.Tags, ",")) + } + if err = tab.Flush(); err != nil { + return err + } + + return nil + }, +} diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index 7fb236bf..4fd2a451 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -1,7 +1,10 @@ package secrets import ( + "fmt" "testing" + + "github.com/zricethezav/gitleaks/v8/config" ) func TestLoadAllRules(t *testing.T) { @@ -12,6 +15,22 @@ func TestLoadAllRules(t *testing.T) { } } +func TestLoadAllRules_DuplicateRuleID(t *testing.T) { + ruleIDMap := make(map[string]bool) + allRules, err := loadAllRules() + if err != nil { + t.Error(err) + } + + for _, rule := range allRules { + if _, ok := ruleIDMap[rule.Rule.RuleID]; ok { + t.Errorf("duplicate rule id found: %s", rule.Rule.RuleID) + } + + ruleIDMap[rule.Rule.RuleID] = true + } +} + func TestIsAllFilter_AllFilterNotPresent(t *testing.T) { filters := []string{"token", "key"} @@ -96,3 +115,242 @@ func TestGetRules_IdAndKeyFilters(t *testing.T) { t.Error("no rules were loaded") } } + +func TestInit(t *testing.T) { + allRules, err := loadAllRules() + if err != nil { + t.Error(err) + } + rulesCount := len(allRules) + + tests := []struct { + name string + includeList []string + excludeList []string + expectedErr error + expectedLen int + }{ + { + name: "include and exclude flags used together", + includeList: []string{"tag1"}, + excludeList: []string{"tag2"}, + expectedErr: fmt.Errorf("cannot use both include and exclude flags"), + expectedLen: 0, + }, + { + name: "non existent include flag", + includeList: []string{"non-existent-tag-name"}, + excludeList: []string{}, + expectedErr: fmt.Errorf("no rules were selected"), + expectedLen: 0, + }, + { + name: "non existent exclude flag", + includeList: []string{}, + excludeList: []string{"non-existent-tag-name"}, + expectedErr: nil, + expectedLen: rulesCount, + }, + { + name: "no flags", + includeList: []string{}, + excludeList: []string{}, + expectedErr: nil, + expectedLen: rulesCount, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + secrets, err := Init(tt.includeList, tt.excludeList) + + if err != nil { + if tt.expectedErr == nil { + t.Errorf("expected no error, but got %s", err) + } else if err.Error() == tt.expectedErr.Error() { + return + } else { + t.Errorf("expected error %s, but got %s", tt.expectedErr, err) + } + } else if tt.expectedErr != nil { + t.Errorf("expected error %s, but got none", tt.expectedErr) + } + + if len(secrets.rules) != tt.expectedLen { + t.Errorf("expected %d rules, but got %d", tt.expectedLen, len(secrets.rules)) + } + }) + } +} + +func TestSelectRules(t *testing.T) { + testCases := []struct { + name string + allRules []Rule + tags []string + expectedResult map[string]config.Rule + }{ + { + name: "No matching tags", + allRules: []Rule{ + createRule("rule1", "tag1", "tag2"), + createRule("rule2", "tag3", "tag4"), + }, + tags: []string{"tag5", "tag6"}, + expectedResult: map[string]config.Rule{}, + }, + { + name: "Matching rule ID", + allRules: []Rule{ + createRule("rule1", "tag1", "tag2"), + createRule("rule2", "tag3", "tag4"), + }, + tags: []string{"rule1"}, + expectedResult: createRules("rule1"), + }, + { + name: "Matching tag", + allRules: []Rule{ + createRule("rule1", "tag1", "tag2"), + createRule("rule2", "tag3", "tag4"), + }, + tags: []string{"tag2"}, + expectedResult: createRules("rule1"), + }, + { + name: "Matching tag and rule ID", + allRules: []Rule{ + createRule("rule1", "tag1", "tag2"), + createRule("rule2", "tag3", "tag4"), + }, + tags: []string{"rule1", "tag2"}, + expectedResult: createRules("rule1"), + }, + { + name: "Matching multiple tags", + allRules: []Rule{ + createRule("rule1", "tag1", "tag2"), + createRule("rule2", "tag3", "tag4"), + createRule("rule3", "tag2", "tag4"), + }, + tags: []string{"tag2", "tag4"}, + expectedResult: createRules("rule1", "rule2", "rule3"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := selectRules(tc.allRules, tc.tags) + + if len(result) != len(tc.expectedResult) { + t.Errorf("Expected %d rules to be applied, but got %d", len(tc.expectedResult), len(result)) + } + + for ruleID, expectedRule := range tc.expectedResult { + if _, ok := result[ruleID]; !ok { + t.Errorf("Expected rule %s to be applied, but it was not", ruleID) + } else { + if result[ruleID].RuleID != expectedRule.RuleID { + t.Errorf("Expected rule %s to have RuleID %s, but it had RuleID %s", ruleID, expectedRule.RuleID, result[ruleID].RuleID) + } + } + } + }) + } +} + +func TestExcludeRules(t *testing.T) { + tests := []struct { + name string + allRules []Rule + tags []string + expectedResult map[string]config.Rule + }{ + { + name: "Empty list", + allRules: []Rule{ + createRule("rule1", "tag1", "tag2"), + createRule("rule2", "tag2", "tag3"), + }, + tags: []string{}, + expectedResult: createRules("rule1", "rule2"), + }, + { + name: "Exclude non-existing tag", + allRules: []Rule{ + createRule("rule1", "tag1", "tag2"), + createRule("rule2", "tag2", "tag3"), + }, + tags: []string{"non-existing-tag"}, + expectedResult: createRules("rule1", "rule2"), + }, + { + name: "Exclude one rule ID", + allRules: []Rule{ + createRule("rule1", "tag1", "tag2"), + createRule("rule2", "tag2", "tag3"), + }, + tags: []string{"rule1"}, + expectedResult: createRules("rule2"), + }, + { + name: "Exclude one tag", + allRules: []Rule{ + createRule("rule1", "tag1", "tag2"), + createRule("rule2", "tag2", "tag3"), + }, + tags: []string{"tag2"}, + expectedResult: map[string]config.Rule{}, + }, + { + name: "Exclude all tags", + allRules: []Rule{ + createRule("rule1", "tag1", "tag2"), + createRule("rule2", "tag2", "tag3"), + }, + tags: []string{"tag1", "tag2", "tag3"}, + expectedResult: map[string]config.Rule{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotResult := excludeRules(tt.allRules, tt.tags) + + if len(gotResult) != len(tt.expectedResult) { + t.Errorf("expected %d rules, but got %d", len(tt.expectedResult), len(gotResult)) + } + + for _, rule := range tt.allRules { + if _, ok := tt.expectedResult[rule.Rule.RuleID]; ok { + if _, ok := gotResult[rule.Rule.RuleID]; !ok { + t.Errorf("expected rule %s to be present, but it was not", rule.Rule.RuleID) + } + } else { + if _, ok := gotResult[rule.Rule.RuleID]; ok { + t.Errorf("expected rule %s to be excluded, but it was not", rule.Rule.RuleID) + } + } + } + }) + } +} + +func createRule(ruleID string, tags ...string) Rule { + return Rule{ + Rule: config.Rule{ + RuleID: ruleID, + }, + Tags: tags, + } +} + +func createRules(ruleIDs ...string) map[string]config.Rule { + rules := make(map[string]config.Rule) + for _, ruleID := range ruleIDs { + rules[ruleID] = config.Rule{ + RuleID: ruleID, + } + } + return rules +}