From 8156dfa1eb31c74665a15532af06ae8f33173c92 Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Wed, 23 Aug 2023 11:49:57 +0300 Subject: [PATCH] feat: catch username password inside urls (#169) Closes #165 After researching the Gitleaks docs, I concluded that adding a toml file won't extend the current rules. This is due to the way this project extends Gitleaks. **Proposed Changes** - added a secrets/custom_rules.go that includes custom rules configuration. - a test to validate custom rules were added to the all rules array. - added a function to append the custom rules to all rules array. I submit this contribution under the Apache-2.0 license. --------- Co-authored-by: Baruch Odem (Rothkoff) Co-authored-by: Baruch Odem --- secrets/rules/authenticated_url.go | 31 +++++++++++++++++++++++++ secrets/rules/rule.go | 37 ++++++++++++++++++++++++++++++ secrets/rules/rule_test.go | 28 ++++++++++++++++++++++ secrets/secrets.go | 2 ++ secrets/secrets_test.go | 25 ++++++++++++++++++++ 5 files changed, 123 insertions(+) create mode 100644 secrets/rules/authenticated_url.go create mode 100644 secrets/rules/rule.go create mode 100644 secrets/rules/rule_test.go diff --git a/secrets/rules/authenticated_url.go b/secrets/rules/authenticated_url.go new file mode 100644 index 00000000..999533f1 --- /dev/null +++ b/secrets/rules/authenticated_url.go @@ -0,0 +1,31 @@ +package rules + +import ( + "regexp" + + "github.com/zricethezav/gitleaks/v8/config" +) + +func AuthenticatedURL() *config.Rule { + regex, _ := regexp.Compile(`:\/\/(.+:.+)?@`) + rule := config.Rule{ + Description: "Identify username:password inside URLS", + RuleID: "authenticated-url", + Regex: regex, + Keywords: []string{}, + SecretGroup: 1, + } + + tPositives := []string{ + "mongodb+srv://radar:mytoken@io.dbb.mongodb.net/?retryWrites=true&w=majority", + "--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs", + "https://abc:123@google.com", + } + + fPositives := []string{ + "https://google.com", + "https://google.com?user=abc&password=123", + } + + return validate(rule, tPositives, fPositives) +} diff --git a/secrets/rules/rule.go b/secrets/rules/rule.go new file mode 100644 index 00000000..59d4b36e --- /dev/null +++ b/secrets/rules/rule.go @@ -0,0 +1,37 @@ +package rules + +import ( + "strings" + + "github.com/rs/zerolog/log" + "github.com/zricethezav/gitleaks/v8/config" + "github.com/zricethezav/gitleaks/v8/detect" +) + +// Copied from https://github.com/gitleaks/gitleaks/blob/463d24618fa42fc7629dc30c9744ebe36c5df1ab/cmd/generate/config/rules/rule.go +func validate(r config.Rule, truePositives []string, falsePositives []string) *config.Rule { + // normalize keywords like in the config package + var keywords []string + for _, k := range r.Keywords { + keywords = append(keywords, strings.ToLower(k)) + } + r.Keywords = keywords + + rules := make(map[string]config.Rule) + rules[r.RuleID] = r + d := detect.NewDetector(config.Config{ + Rules: rules, + Keywords: keywords, + }) + for _, tp := range truePositives { + if len(d.DetectString(tp)) != 1 { + log.Fatal().Msgf("Failed to validate. For rule ID [%s], true positive [%s] was not detected by regexp [%s]", r.RuleID, tp, r.Regex) + } + } + for _, fp := range falsePositives { + if len(d.DetectString(fp)) != 0 { + log.Fatal().Msgf("Failed to validate. For rule ID [%s], false positive [%s] was detected by regexp [%s]", r.RuleID, fp, r.Regex) + } + } + return &r +} diff --git a/secrets/rules/rule_test.go b/secrets/rules/rule_test.go new file mode 100644 index 00000000..34e27da3 --- /dev/null +++ b/secrets/rules/rule_test.go @@ -0,0 +1,28 @@ +package rules_test + +import ( + "testing" + + "github.com/checkmarx/2ms/secrets/rules" + "github.com/zricethezav/gitleaks/v8/config" +) + +func Test2msRules(t *testing.T) { + t.Parallel() + + testsRules := []struct { + name string + validate func() *config.Rule + }{ + {name: "AuthenticatedURL", validate: rules.AuthenticatedURL}, + } + + for _, tRule := range testsRules { + testRule := tRule // fix for loop variable being captured by func literal + t.Run(testRule.name, func(t *testing.T) { + t.Parallel() + + testRule.validate() + }) + } +} diff --git a/secrets/secrets.go b/secrets/secrets.go index 3a6d720d..11c17a9d 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -11,6 +11,7 @@ import ( "github.com/checkmarx/2ms/plugins" "github.com/checkmarx/2ms/reporting" + internalRules "github.com/checkmarx/2ms/secrets/rules" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" @@ -377,6 +378,7 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.YandexAWSAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.YandexAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.ZendeskSecretKey(), Tags: []string{TagSecretKey}}) + allRules = append(allRules, Rule{Rule: *internalRules.AuthenticatedURL(), Tags: []string{TagSensitiveUrl}}) return allRules, nil } diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index a9ec501c..7d662320 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -364,6 +364,31 @@ func TestSecrets(t *testing.T) { Name string ShouldFind bool }{ + { + Content: "", + Name: "empty", + ShouldFind: false, + }, + { + Content: "mongodb+srv://radar:mytoken@io.dbb.mongodb.net/?retryWrites=true&w=majority", + Name: "Authenticated URL", + ShouldFind: true, + }, + { + Content: "--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs", + Name: "Authenticated URL", + ShouldFind: true, + }, + { + Content: "https://abc:123@google.com", + Name: "Basic Authenticated URL", + ShouldFind: true, + }, + { + Content: "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", + Name: "GitHub Personal Access Token", + ShouldFind: true, + }, { Content: "AKCp8jRRiQSAbghbuZmHKZcaKGEqbAASGH2SAb3rxXJQsSq9dGga8gFXe6aHpcRmzuHxN6oaT", Name: "JFROG Secret without keyword",