fix: obfuscate secret() calls, resolve sensitive config

cmd/testworkflow-init/constants/commands.go

@@ -3,6 +3,7 @@ package constants
 const (
 	EnvGroupActions = "01"
 	EnvGroupDebug   = "00"
+	EnvGroupSecrets = "02"
 
 	EnvNodeName           = "TKI_N"
 	EnvPodName            = "TKI_P"

cmd/testworkflow-init/orchestration/setup.go

@@ -17,7 +17,7 @@ import (
 )
 
 var (
-	scopedRegex              = regexp.MustCompile(`^_(00|01|\d|[1-9]\d*)(C)?(S?)_`)
+	scopedRegex              = regexp.MustCompile(`^_(00|01|02|\d|[1-9]\d*)(C)?(S?)_`)
 	Setup                    = newSetup()
 	defaultWorkingDir        = getWorkingDir()
 	commonSensitiveVariables = []string{
@@ -130,6 +130,16 @@ func (c *setup) GetSensitiveWords() []string {
 			words = append(words, value)
 		}
 	}
+	// TODO(TKC-2585): Avoid adding the secrets to all the groups without isolation
+	for k := range c.envGroups[constants.EnvGroupSecrets] {
+		value := os.Getenv(k)
+		if len(value) < c.minSensitiveWordLength {
+			continue
+		}
+		if _, ok := c.envGroupsSensitive[constants.EnvGroupSecrets][k]; ok {
+			words = append(words, value)
+		}
+	}
 	return words
 }
 
@@ -159,6 +169,15 @@ func (c *setup) UseEnv(group string) {
 		}
 	}
 
+	// TODO(TKC-2585): Avoid adding the secrets to all the groups without isolation
+	for k, v := range c.envGroups[constants.EnvGroupSecrets] {
+		if _, ok := c.envGroupsComputed[constants.EnvGroupSecrets][k]; ok {
+			envTemplates[k] = v
+		} else {
+			os.Setenv(k, v)
+		}
+	}
+
 	// Configure PWD variable, to make it similar to shell environment variables
 	cwd := getWorkingDir()
 	if os.Getenv("PWD") == "" {

pkg/expressions/libs/fs_test.go

@@ -9,47 +9,28 @@ import (
 	"github.com/kubeshop/testkube/pkg/expressions"
 )
 
-func MustCall(m expressions.Machine, name string, args ...interface{}) interface{} {
-	list := make([]expressions.StaticValue, len(args))
-	for i, v := range args {
-		if vv, ok := v.(expressions.StaticValue); ok {
-			list[i] = vv
-		} else {
-			list[i] = expressions.NewValue(v)
-		}
-	}
-	v, ok, err := m.Call(name, list...)
-	if err != nil {
-		panic(err)
-	}
-	if !ok {
-		panic("not recognized")
-	}
-	return v.Static().Value()
-}
-
 func TestFsLibGlob(t *testing.T) {
 	fsys := &afero.IOFS{Fs: afero.NewMemMapFs()}
 	_ = afero.WriteFile(fsys.Fs, "etc/file1.txt", nil, 0644)
 	_ = afero.WriteFile(fsys.Fs, "else/file1.txt", nil, 0644)
 	_ = afero.WriteFile(fsys.Fs, "another-file.txt", nil, 0644)
 	_ = afero.WriteFile(fsys.Fs, "etc/nested/file2.json", nil, 0644)
 	machine := NewFsMachine(fsys, "/etc")
-	assert.Equal(t, []string{"/etc/file1.txt", "/etc/nested/file2.json"}, MustCall(machine, "glob", "**/*"))
-	assert.Equal(t, []string{"/etc/file1.txt"}, MustCall(machine, "glob", "*"))
-	assert.Equal(t, []string{"/etc/nested/file2.json"}, MustCall(machine, "glob", "**/*.json"))
-	assert.Equal(t, []string{"/etc/file1.txt", "/etc/nested/file2.json"}, MustCall(machine, "glob", "**/*.json", "*.txt"))
-	assert.Equal(t, []string{"/another-file.txt", "/else/file1.txt", "/etc/file1.txt"}, MustCall(machine, "glob", "/**/*.txt"))
-	assert.Equal(t, []string{"/another-file.txt", "/etc/file1.txt"}, MustCall(machine, "glob", "/**/*.txt", "!/else/**/*"))
+	assert.Equal(t, []string{"/etc/file1.txt", "/etc/nested/file2.json"}, expressions.MustCall(machine, "glob", "**/*"))
+	assert.Equal(t, []string{"/etc/file1.txt"}, expressions.MustCall(machine, "glob", "*"))
+	assert.Equal(t, []string{"/etc/nested/file2.json"}, expressions.MustCall(machine, "glob", "**/*.json"))
+	assert.Equal(t, []string{"/etc/file1.txt", "/etc/nested/file2.json"}, expressions.MustCall(machine, "glob", "**/*.json", "*.txt"))
+	assert.Equal(t, []string{"/another-file.txt", "/else/file1.txt", "/etc/file1.txt"}, expressions.MustCall(machine, "glob", "/**/*.txt"))
+	assert.Equal(t, []string{"/another-file.txt", "/etc/file1.txt"}, expressions.MustCall(machine, "glob", "/**/*.txt", "!/else/**/*"))
 }
 
 func TestFsLibRead(t *testing.T) {
 	fsys := &afero.IOFS{Fs: afero.NewMemMapFs()}
 	_ = afero.WriteFile(fsys.Fs, "etc/file1.txt", []byte("foo"), 0644)
 	_ = afero.WriteFile(fsys.Fs, "another-file.txt", []byte("bar"), 0644)
 	machine := NewFsMachine(fsys, "/etc")
-	assert.Equal(t, "foo", MustCall(machine, "file", "file1.txt"))
-	assert.Equal(t, "foo", MustCall(machine, "file", "/etc/file1.txt"))
-	assert.Equal(t, "bar", MustCall(machine, "file", "../another-file.txt"))
-	assert.Equal(t, "bar", MustCall(machine, "file", "/another-file.txt"))
+	assert.Equal(t, "foo", expressions.MustCall(machine, "file", "file1.txt"))
+	assert.Equal(t, "foo", expressions.MustCall(machine, "file", "/etc/file1.txt"))
+	assert.Equal(t, "bar", expressions.MustCall(machine, "file", "../another-file.txt"))
+	assert.Equal(t, "bar", expressions.MustCall(machine, "file", "/another-file.txt"))
 }

pkg/expressions/utils.go

@@ -85,3 +85,22 @@ func Escape(str string) string {
 func EscapeLabelKeyForVarName(key string) string {
 	return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(key, ".", "_"), "-", "_"), "/", "_")
 }
+
+func MustCall(m Machine, name string, args ...interface{}) interface{} {
+	list := make([]StaticValue, len(args))
+	for i, v := range args {
+		if vv, ok := v.(StaticValue); ok {
+			list[i] = vv
+		} else {
+			list[i] = NewValue(v)
+		}
+	}
+	v, ok, err := m.Call(name, list...)
+	if err != nil {
+		panic(err)
+	}
+	if !ok {
+		panic("not recognized")
+	}
+	return v.Static().Value()
+}

pkg/testworkflows/testworkflowprocessor/processor.go

@@ -15,7 +15,6 @@ import (
 	testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1"
 	"github.com/kubeshop/testkube/internal/common"
 	"github.com/kubeshop/testkube/pkg/expressions"
-	"github.com/kubeshop/testkube/pkg/expressions/libs"
 	"github.com/kubeshop/testkube/pkg/imageinspector"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes"
@@ -112,7 +111,7 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo
 		AppendVolumeMounts(layer.AddEmptyDirVolume(nil, constants.DefaultDataPath))
 
 	mapEnv := make(map[string]corev1.EnvVarSource)
-	extendedMachines := append(machines, libs.NewSecretMachine(mapEnv))
+	extendedMachines := append(machines, createSecretMachine(mapEnv))
 
 	// Fetch resource root and resource ID
 	resourceRoot, err := expressions.EvalExpression("resource.root", extendedMachines...)
@@ -395,6 +394,7 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo
 	}
 	jobSpec.Spec.Template = podSpec
 
+	// TODO(TKC-2585): Avoid adding the secrets to all the groups without isolation
 	addEnvVarToContainerSpec(mapEnv, jobSpec.Spec.Template.Spec.InitContainers)
 	addEnvVarToContainerSpec(mapEnv, jobSpec.Spec.Template.Spec.Containers)
 

pkg/testworkflows/testworkflowprocessor/secretmachine.go

@@ -0,0 +1,55 @@
+package testworkflowprocessor
+
+import (
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/kubeshop/testkube/cmd/testworkflow-init/constants"
+	"github.com/kubeshop/testkube/pkg/expressions"
+	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes"
+)
+
+func createSecretMachine(mapEnvs map[string]corev1.EnvVarSource) expressions.Machine {
+	return expressions.NewMachine().
+		RegisterFunction("secret", func(values ...expressions.StaticValue) (interface{}, bool, error) {
+			computed := false
+			if len(values) == 3 {
+				if values[2].IsBool() {
+					computed, _ = values[2].BoolValue()
+				} else {
+					return nil, true, fmt.Errorf(`"secret" function expects 3rd argument to be boolean, %s provided`, values[3].String())
+				}
+			} else if len(values) != 2 {
+				return nil, true, fmt.Errorf(`"secret" function expects 2-3 arguments, %d provided`, len(values))
+			}
+
+			secretName, _ := values[0].StringValue()
+			keyName, _ := values[1].StringValue()
+			strs := []string{secretName, keyName}
+			for i := range strs {
+				j := 0
+				for _, char := range []string{"-", "."} {
+					for ; strings.Contains(strs[i], char); j++ {
+						strs[i] = strings.Replace(strs[i], char, fmt.Sprintf("_%d_", j), 1)
+					}
+				}
+			}
+
+			// TODO(TKC-2585): Avoid adding the secrets to all the groups with virtual 02 group
+			envName := fmt.Sprintf("%s_K_%s", strs[0], strs[1])
+			internalName := actiontypes.EnvName(constants.EnvGroupSecrets, computed, true, envName)
+			mapEnvs[internalName] = corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: secretName,
+					},
+					Key: keyName,
+				},
+			}
+
+			return expressions.NewValue(fmt.Sprintf("{{%senv.%s}}", expressions.InternalFnCall, envName)), true, nil
+		})
+
+}

pkg/testworkflows/testworkflowprocessor/secretmachine_test.go

@@ -0,0 +1,42 @@
+package testworkflowprocessor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/kubeshop/testkube/pkg/expressions"
+)
+
+func TestSecret(t *testing.T) {
+	mapEnvs := make(map[string]corev1.EnvVarSource)
+	machine := createSecretMachine(mapEnvs)
+	assert.Equal(t, "{{"+expressions.InternalFnCall+"env.name_0_one_1_two_K_key_0_three_1_four}}", expressions.MustCall(machine, "secret", "name-one.two", "key-three.four"))
+	assert.EqualValues(t, map[string]corev1.EnvVarSource{
+		"_02S_name_0_one_1_two_K_key_0_three_1_four": {
+			SecretKeyRef: &corev1.SecretKeySelector{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: "name-one.two",
+				},
+				Key: "key-three.four",
+			},
+		},
+	}, mapEnvs)
+}
+
+func TestSecretComputed(t *testing.T) {
+	mapEnvs := make(map[string]corev1.EnvVarSource)
+	machine := createSecretMachine(mapEnvs)
+	assert.Equal(t, "{{"+expressions.InternalFnCall+"env.name_0_one_1_two_K_key_0_three_1_four}}", expressions.MustCall(machine, "secret", "name-one.two", "key-three.four", true))
+	assert.EqualValues(t, map[string]corev1.EnvVarSource{
+		"_02CS_name_0_one_1_two_K_key_0_three_1_four": {
+			SecretKeyRef: &corev1.SecretKeySelector{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: "name-one.two",
+				},
+				Key: "key-three.four",
+			},
+		},
+	}, mapEnvs)
+}

pkg/testworkflows/testworkflowresolver/config.go

@@ -78,12 +78,12 @@ func createConfigMachine(cfg map[string]intstr.IntOrString, schema map[string]te
 
 func getSecretCallExpression(expr expressions.Expression, k string, externalize func(key, value string) (*corev1.EnvVarSource, error)) (
 	expressions.Expression, error) {
-	envVar, err := externalize(k, expr.String())
+	envVar, err := externalize(k, expr.Template())
 	if err != nil {
 		return nil, errors.Wrap(err, "config."+k)
 	}
 	if envVar.SecretKeyRef != nil {
-		expr = expressions.NewValue(fmt.Sprintf("{{%ssecret(\"%s\", \"%s\")}}", expressions.InternalFnCall,
+		expr = expressions.NewValue(fmt.Sprintf("{{%ssecret(\"%s\", \"%s\", true)}}", expressions.InternalFnCall,
 			envVar.SecretKeyRef.Name, envVar.SecretKeyRef.Key))
 	}