Skip to content

Commit

Permalink
feat(compute/secrets): redact common org secrets (#1069)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Integralist
Integralist authored Nov 3, 2023
1 parent b6812c8 commit 30ed391
Show file tree
Hide file tree
Showing 3 changed files with 155 additions and 54 deletions.
19 changes: 7 additions & 12 deletions pkg/commands/compute/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -349,13 +349,16 @@ func (c *BuildCommand) AnnotateWasmBinaryLong(wasmtools string, args []string, l
var ms runtime.MemStats
runtime.ReadMemStats(&ms)

// Allow customer to specify their own env variables to be filtered.
ExtendStaticSecretEnvVars(c.MetadataFilterEnvVars)

dc := DataCollection{
ScriptInfo: DataCollectionScriptInfo{
DefaultBuildUsed: language.DefaultBuildScript(),
BuildScript: c.Globals.Manifest.File.Scripts.Build,
EnvVars: c.Globals.Manifest.File.Scripts.EnvVars,
PostInitScript: c.Globals.Manifest.File.Scripts.PostInit,
PostBuildScript: c.Globals.Manifest.File.Scripts.PostBuild,
BuildScript: FilterSecretsFromString(c.Globals.Manifest.File.Scripts.Build),
EnvVars: FilterSecretsFromSlice(c.Globals.Manifest.File.Scripts.EnvVars),
PostInitScript: FilterSecretsFromString(c.Globals.Manifest.File.Scripts.PostInit),
PostBuildScript: FilterSecretsFromString(c.Globals.Manifest.File.Scripts.PostBuild),
},
}

Expand All @@ -380,18 +383,10 @@ func (c *BuildCommand) AnnotateWasmBinaryLong(wasmtools string, args []string, l
}
}

// Allow customer to specify their own env variables to be filtered.
ExtendEnvVarSecretsFilter(c.MetadataFilterEnvVars)

// Filter environment variables using combination of user provided filters and
// the CLI hard-coded filters.
FilterEnvVarSecretsFromSlice(dc.ScriptInfo.EnvVars)

data, err := json.Marshal(dc)
if err != nil {
return err
}
data = FilterEnvVarSecretsFromBytes(data)

args = append(args, fmt.Sprintf("--processed-by=fastly_data=%s", data))

Expand Down
14 changes: 5 additions & 9 deletions pkg/commands/compute/language_toolchain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,21 +89,17 @@ func (bt BuildToolchain) Build() error {
cmd, args := bt.buildFn(bt.buildScript)

if bt.verbose {
text.Description(bt.out, "Build script to execute", fmt.Sprintf("%s %s", cmd, strings.Join(args, " ")))
buildScript := fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))
text.Description(bt.out, "Build script to execute", FilterSecretsFromString(buildScript))

// IMPORTANT: We filter secrets the best we can before printing env vars.
// We use two separate processes to do this.
// First is filtering based on known environment variables.
// Second is filtering based on a generalised regex pattern.
if len(bt.env) > 0 {
envVars := make([]string, len(bt.env))
copy(envVars, bt.env)
ExtendEnvVarSecretsFilter(bt.metadataFilterEnvVars)
FilterEnvVarSecretsFromSlice(envVars)

s := strings.Join(envVars, " ")
s = FilterEnvVarSecretsFromString(s)
text.Description(bt.out, "Build environment variables set", s)
ExtendStaticSecretEnvVars(bt.metadataFilterEnvVars)
s := strings.Join(bt.env, " ")
text.Description(bt.out, "Build environment variables set", FilterSecretsFromString(s))
}
}

Expand Down
176 changes: 143 additions & 33 deletions pkg/commands/compute/secrets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,14 @@ import (
"strings"
)

// filterSecretsPattern attempts to capture a secret assigned in an environment
// variable where the key follows a common pattern.
// https://regex101.com/r/4GnH3r/1
const filterSecretsPattern = `(?i)\b[^\s_]+_(?:API|CLIENTSECRET|CREDENTIALS|KEY|PASSWORD|SECRET|TOKEN)(?:[^=]+)?=(?:\s+)?"?([^\s"]+)` // #nosec G101 (CWE-798)

// filterEnvVarSecrets identify environment variables containing secrets.
var filterEnvVarSecrets = []string{
// StaticSecretEnvVars is a static list of env vars containing secrets.
//
// NOTE: Env Vars pulled from https://github.com/Puliczek/awesome-list-of-secrets-in-environment-variables
//
// The reason for not listing more environment variables is because we have a
// generalised pattern `SecretGeneralisedEnvVarPattern` that catches the
// majority of formats used.
var StaticSecretEnvVars = []string{
"AZURE_CLIENT_ID",
"CI_JOB_JWT",
"CI_JOB_JWT_V2",
Expand All @@ -22,73 +23,182 @@ var filterEnvVarSecrets = []string{
"OKTA_OAUTH2_CLIENTID",
}

// ExtendEnvVarSecretsFilter mutates filterEnvVarSecrets to include user
// SecretGeneralisedEnvVarPattern attempts to capture a secret assigned in an environment
// variable where the key follows a common pattern.
//
// Example:
// https://regex101.com/r/mf9Ymb/1
var SecretGeneralisedEnvVarPattern = regexp.MustCompile(`(?i)\b[^\s]+_(?:API|CLIENTSECRET|CREDENTIALS|KEY|PASSWORD|SECRET|TOKEN)(?:[^=]+)?=(?:\s+)?"?([^\s"]+)`) // #nosec G101 (CWE-798)

// AWSIDPattern is the pattern for an AWS ID.
var AWSIDPattern = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)

// AWSSecretPattern is the pattern for an AWS Secret.
var AWSSecretPattern = regexp.MustCompile(`[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}`)

// GitHubOAuthTokenPattern is the pattern for a GitHub OAuth token.
var GitHubOAuthTokenPattern = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr|github_pat)_[a-zA-Z0-9_]{36,255})\b`)

// GitHubOldOAuthTokenPattern is the pattern for an older GitHub OAuth token format.
var GitHubOldOAuthTokenPattern = regexp.MustCompile(`(?i)(?:github|gh|pat|token)[^\.].{0,40}[ =:'"]+([a-f0-9]{40})\b`)

// GitHubOAuth2ClientIDPattern is the pattern for a GitHub OAuth2 ClientID.
var GitHubOAuth2ClientIDPattern = regexp.MustCompile(`(?i)(?:github)(?:.|[\n\r]){0,40}\b([a-f0-9]{20})\b`)

// GitHubOAuth2ClientSecretPattern is the pattern for a GitHub OAuth2 ClientID.
var GitHubOAuth2ClientSecretPattern = regexp.MustCompile(`(?i)(?:github)(?:.|[\n\r]){0,40}\b([a-f0-9]{40})\b`)

// GitHubAppIDPattern is the pattern for a GitHub App ID.
var GitHubAppIDPattern = regexp.MustCompile(`(?i)(?:github)(?:.|[\n\r]){0,40}\b([0-9]{6})\b`)

// GitHubAppKeyPattern is the pattern for a GitHub App Key.
var GitHubAppKeyPattern = regexp.MustCompile(`(?i)(?:github)(?:.|[\n\r]){0,40}(-----BEGIN RSA PRIVATE KEY-----\s[A-Za-z0-9+\/\s]*\s-----END RSA PRIVATE KEY-----)`)

// SecretPatterns is a collection of secret identifying patterns.
//
// NOTE: Patterns pulled from https://github.com/trufflesecurity/trufflehog
var SecretPatterns = []*regexp.Regexp{
AWSIDPattern,
AWSSecretPattern,
GitHubOAuthTokenPattern,
GitHubOldOAuthTokenPattern,
GitHubOAuth2ClientIDPattern,
GitHubOAuth2ClientSecretPattern,
GitHubAppIDPattern,
GitHubAppKeyPattern,
}

// ExtendStaticSecretEnvVars mutates `StaticSecretEnvVars` to include user
// specified environment variables. The `filter` argument is comma-separated.
func ExtendEnvVarSecretsFilter(filter string) {
func ExtendStaticSecretEnvVars(filter string) {
customFilters := strings.Split(filter, ",")
for _, v := range customFilters {
if v == "" {
continue
}
var found bool
for _, f := range filterEnvVarSecrets {
for _, f := range StaticSecretEnvVars {
if f == v {
found = true
break
}
}
if !found {
filterEnvVarSecrets = append(filterEnvVarSecrets, v)
StaticSecretEnvVars = append(StaticSecretEnvVars, v)
}
}
}

// FilterEnvVarSecretsFromSlice mutates the input data such that any value
// FilterSecretsFromSlice returns the input slice modified such that any value
// assigned to an environment variable (identified as containing a secret) is
// redacted. Additionally, any 'value' identified as being a secret will also be
// redacted.
func FilterEnvVarSecretsFromSlice(data []string) {
for i, v := range data {
for _, f := range filterEnvVarSecrets {
k := strings.Split(v, "=")[0]
if strings.HasPrefix(k, f) {
data[i] = k + "=REDACTED"
//
// NOTE: `data` is expected to contain "KEY=VALUE" formatted strings.
func FilterSecretsFromSlice(data []string) []string {
copyOfData := make([]string, len(data))
copy(copyOfData, data)

for i, keypair := range copyOfData {
k, v, found := strings.Cut(keypair, "=")
if !found {
return copyOfData
}
for _, f := range StaticSecretEnvVars {
if k == f {
copyOfData[i] = k + "=REDACTED"
break
}
}
if strings.Contains(copyOfData[i], "REDACTED") {
continue
}
for _, matches := range SecretGeneralisedEnvVarPattern.FindAllStringSubmatch(keypair, -1) {
if len(matches) == 2 {
o := matches[0]
n := strings.ReplaceAll(matches[0], matches[1], "REDACTED")
copyOfData[i] = strings.ReplaceAll(keypair, o, n)
}
}
if strings.Contains(copyOfData[i], "REDACTED") {
continue
}
for _, pattern := range SecretPatterns {
n := pattern.ReplaceAllString(v, "REDACTED")
copyOfData[i] = k + "=" + n
if n == "REDACTED" {
break
}
}
}

return copyOfData
}

// FilterEnvVarSecretsFromBytes mutates the input data such that any value
// FilterSecretsFromString returns the input string modified such that any value
// assigned to an environment variable (identified as containing a secret) is
// redacted. Additionally, any 'value' identified as being a secret will also be
// redacted.
//
// Example:
// https://go.dev/play/p/GYXMNc7Froz
func FilterEnvVarSecretsFromBytes(data []byte) []byte {
re := regexp.MustCompile(filterSecretsPattern)
for _, matches := range re.FindAllSubmatch(data, -1) {
// https://go.dev/play/p/jhCcC4SlsHA
//
// NOTE: The input data should be simple (i.e. not a complex json object).
// Otherwise the `SecretGeneralisedEnvVarPattern` will unlikely match all cases.
func FilterSecretsFromString(data string) string {
staticSecretEnvVarsPattern := regexp.MustCompile(`(?i)\b(?:` + strings.Join(StaticSecretEnvVars, "|") + `)(?:\s+)?=(?:\s+)?"?([^\s"]+)`)
for _, matches := range staticSecretEnvVarsPattern.FindAllStringSubmatch(data, -1) {
if len(matches) == 2 {
o := matches[0]
n := bytes.ReplaceAll(matches[0], matches[1], []byte("REDACTED"))
data = bytes.ReplaceAll(data, o, n)
n := strings.ReplaceAll(matches[0], matches[1], "REDACTED")
data = strings.ReplaceAll(data, o, n)
}
}
for _, matches := range SecretGeneralisedEnvVarPattern.FindAllStringSubmatch(data, -1) {
if len(matches) == 2 {
o := matches[0]
n := strings.ReplaceAll(matches[0], matches[1], "REDACTED")
data = strings.ReplaceAll(data, o, n)
}
}
for _, pattern := range SecretPatterns {
data = pattern.ReplaceAllString(data, "REDACTED")
}
return data
}

// FilterEnvVarSecretsFromString mutates the input data such that any value
// FilterSecretsFromBytes returns the input string modified such that any value
// assigned to an environment variable (identified as containing a secret) is
// redacted. Additionally, any 'value' identified as being a secret will also be
// redacted.
//
// Example:
// https://go.dev/play/p/GYXMNc7Froz
func FilterEnvVarSecretsFromString(data string) string {
re := regexp.MustCompile(filterSecretsPattern)
for _, matches := range re.FindAllStringSubmatch(data, -1) {
// https://go.dev/play/p/jhCcC4SlsHA
//
// NOTE: The input data should be simple (i.e. not a complex json object).
// Otherwise the `SecretGeneralisedEnvVarPattern` will unlikely match all cases.
func FilterSecretsFromBytes(data []byte) []byte {
copyOfData := make([]byte, len(data))
copy(copyOfData, data)

staticSecretEnvVarsPattern := regexp.MustCompile(`(?i)\b(?:` + strings.Join(StaticSecretEnvVars, "|") + `)(?:\s+)?=(?:\s+)?"?([^\s"]+)`)
for _, matches := range staticSecretEnvVarsPattern.FindAllSubmatch(copyOfData, -1) {
if len(matches) == 2 {
o := matches[0]
n := strings.ReplaceAll(matches[0], matches[1], "REDACTED")
data = strings.ReplaceAll(data, o, n)
n := bytes.ReplaceAll(matches[0], matches[1], []byte("REDACTED"))
copyOfData = bytes.ReplaceAll(copyOfData, o, n)
}
}
return data
for _, matches := range SecretGeneralisedEnvVarPattern.FindAllSubmatch(copyOfData, -1) {
if len(matches) == 2 {
o := matches[0]
n := bytes.ReplaceAll(matches[0], matches[1], []byte("REDACTED"))
copyOfData = bytes.ReplaceAll(copyOfData, o, n)
}
}
for _, pattern := range SecretPatterns {
copyOfData = pattern.ReplaceAll(copyOfData, []byte("REDACTED"))
}

return copyOfData
}

0 comments on commit 30ed391

Please sign in to comment.