From 1eaac1c72dedf800a40e3dc1eac7cb9287443903 Mon Sep 17 00:00:00 2001 From: Mahesh Hadimani Date: Thu, 13 Aug 2020 14:34:23 +0530 Subject: [PATCH 1/4] Added support for IRSA, fixes - #86 --- agent-inject/agent/agent.go | 66 ++++++++++++++---- agent-inject/agent/agent_test.go | 31 +++++++++ agent-inject/agent/container_env.go | 17 +++++ agent-inject/agent/container_env_test.go | 55 ++++++++++++++- agent-inject/agent/container_init_sidecar.go | 45 ++++++++++--- agent-inject/agent/container_sidecar.go | 44 +++++++++--- agent-inject/agent/container_sidecar_test.go | 70 ++++++++++++++++++++ 7 files changed, 292 insertions(+), 36 deletions(-) diff --git a/agent-inject/agent/agent.go b/agent-inject/agent/agent.go index 0957089f..0e2666cb 100644 --- a/agent-inject/agent/agent.go +++ b/agent-inject/agent/agent.go @@ -123,6 +123,15 @@ type Agent struct { // SetSecurityContext controls whether the injected containers have a // SecurityContext set. SetSecurityContext bool + + // AwsIamTokenAccountName is the aws iam volume mount name for the pod. + // Need this for IRSA aka pod identity + AwsIamTokenAccountName string + + // AwsIamTokenAccountPath is the aws iam volume mount path for the pod + // where the JWT would be present + // Need this for IRSA aka pod identity + AwsIamTokenAccountPath string } type Secret struct { @@ -213,21 +222,24 @@ type VaultAgentCache struct { // New creates a new instance of Agent by parsing all the Kubernetes annotations. func New(pod *corev1.Pod, patches []*jsonpatch.JsonPatchOperation) (*Agent, error) { saName, saPath := serviceaccount(pod) + iamName, iamPath := getAwsIamTokenAccount(pod) agent := &Agent{ - Annotations: pod.Annotations, - ConfigMapName: pod.Annotations[AnnotationAgentConfigMap], - ImageName: pod.Annotations[AnnotationAgentImage], - LimitsCPU: pod.Annotations[AnnotationAgentLimitsCPU], - LimitsMem: pod.Annotations[AnnotationAgentLimitsMem], - Namespace: pod.Annotations[AnnotationAgentRequestNamespace], - Patches: patches, - Pod: pod, - RequestsCPU: pod.Annotations[AnnotationAgentRequestsCPU], - RequestsMem: pod.Annotations[AnnotationAgentRequestsMem], - ServiceAccountName: saName, - ServiceAccountPath: saPath, - Status: pod.Annotations[AnnotationAgentStatus], + Annotations: pod.Annotations, + ConfigMapName: pod.Annotations[AnnotationAgentConfigMap], + ImageName: pod.Annotations[AnnotationAgentImage], + LimitsCPU: pod.Annotations[AnnotationAgentLimitsCPU], + LimitsMem: pod.Annotations[AnnotationAgentLimitsMem], + Namespace: pod.Annotations[AnnotationAgentRequestNamespace], + Patches: patches, + Pod: pod, + RequestsCPU: pod.Annotations[AnnotationAgentRequestsCPU], + RequestsMem: pod.Annotations[AnnotationAgentRequestsMem], + ServiceAccountName: saName, + ServiceAccountPath: saPath, + AwsIamTokenAccountName: iamName, + AwsIamTokenAccountPath: iamPath, + Status: pod.Annotations[AnnotationAgentStatus], Vault: Vault{ Address: pod.Annotations[AnnotationVaultService], AuthPath: pod.Annotations[AnnotationVaultAuthPath], @@ -508,6 +520,34 @@ func serviceaccount(pod *corev1.Pod) (string, string) { return serviceAccountName, serviceAccountPath } +// IRSA support - get aws_iam_token volume mount details to inject to vault containers +func getAwsIamTokenAccount(pod *corev1.Pod) (string, string) { + var awsIamTokenAccountName, awsIamTokenAccountPath string + for _, container := range pod.Spec.Containers { + for _, volumes := range container.VolumeMounts { + if strings.Contains(volumes.MountPath, "eks.amazonaws.com") { + return volumes.Name, volumes.MountPath + } + } + } + return awsIamTokenAccountName, awsIamTokenAccountPath +} + +// IRSA support - get aws envs to inject to vault containers +func (a *Agent) getEnvsFromContainer(pod *corev1.Pod) map[string]string { + envMap := make(map[string]string) + for _, container := range pod.Spec.Containers { + for _, env := range container.Env { + if strings.Contains(env.Name, "AWS_ROLE_ARN") || strings.Contains(env.Name, "AWS_WEB_IDENTITY_TOKEN_FILE") { + if _, ok := envMap[env.Name]; !ok { + envMap[env.Name] = env.Value + } + } + } + } + return envMap +} + func (a *Agent) vaultCliFlags() []string { flags := []string{ fmt.Sprintf("-address=%s", a.Vault.Address), diff --git a/agent-inject/agent/agent_test.go b/agent-inject/agent/agent_test.go index d99117e3..47d60c42 100644 --- a/agent-inject/agent/agent_test.go +++ b/agent-inject/agent/agent_test.go @@ -29,6 +29,37 @@ func testPod(annotations map[string]string) *corev1.Pod { } } +func testPodIRSA(annotations map[string]string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foobar", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "foobar", + MountPath: "serviceaccount/somewhere", + }, + }, + }, + { + Name: "aws-iam-token", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "aws-iam-token", + MountPath: "/var/run/secrets/eks.amazonaws.com/serviceaccount", + }, + }, + }, + }, + }, + } +} + func TestShouldInject(t *testing.T) { tests := []struct { annotations map[string]string diff --git a/agent-inject/agent/container_env.go b/agent-inject/agent/container_env.go index 21493f86..75709f63 100644 --- a/agent-inject/agent/container_env.go +++ b/agent-inject/agent/container_env.go @@ -2,6 +2,8 @@ package agent import ( "encoding/base64" + "log" + corev1 "k8s.io/api/core/v1" ) @@ -42,6 +44,21 @@ func (a *Agent) ContainerEnvVars(init bool) ([]corev1.EnvVar, error) { Name: "VAULT_CONFIG", Value: b64Config, }) + + // Add IRSA AWS Env variables for vault containers + if a.Pod != nil { + envMap := a.getEnvsFromContainer(a.Pod) + if len(envMap) == 0 || len(envMap) >= 2 { + for k, v := range envMap { + envs = append(envs, corev1.EnvVar{ + Name: k, + Value: v, + }) + } + } else { + log.Println("WARN: Could not find 'AWS ROLE/ AWS_WEB_IDENTITY_TOKEN_FILE' env variables") + } + } } return envs, nil diff --git a/agent-inject/agent/container_env_test.go b/agent-inject/agent/container_env_test.go index 233b93b8..c9017d9f 100644 --- a/agent-inject/agent/container_env_test.go +++ b/agent-inject/agent/container_env_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/hashicorp/vault/sdk/helper/strutil" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestContainerEnvs(t *testing.T) { @@ -25,7 +27,6 @@ func TestContainerEnvs(t *testing.T) { if err != nil { t.Errorf("got error, shouldn't have: %s", err) } - if len(envs) != len(tt.expectedEnvs) { t.Errorf("number of envs mismatch, wanted %d, got %d", len(tt.expectedEnvs), len(envs)) } @@ -37,3 +38,55 @@ func TestContainerEnvs(t *testing.T) { } } } + +func TestContainerEnvsForIRSA(t *testing.T) { + envTests := []struct { + agent Agent + expectedEnvs []string + }{ + {Agent{Pod: testPodWithoutIRSA()}, []string{"VAULT_CONFIG"}}, + {Agent{Pod: testPodWithIRSA()}, []string{"VAULT_CONFIG", "AWS_ROLE_ARN", "AWS_WEB_IDENTITY_TOKEN_FILE"}}, + } + for _, tt := range envTests { + envs, err := tt.agent.ContainerEnvVars(true) + if err != nil { + t.Errorf("got error, shouldn't have: %s", err) + } + if len(envs) != len(tt.expectedEnvs) { + t.Errorf("number of envs mismatch, wanted %d, got %d", len(tt.expectedEnvs), len(envs)) + } + } +} + +func testPodWithoutIRSA() *corev1.Pod { + return testPodWithEnv(nil) +} + +func testPodWithIRSA() *corev1.Pod { + return testPodWithEnv([]corev1.EnvVar{ + { + Name: "AWS_ROLE_ARN", + Value: "foorole", + }, + { + Name: "AWS_WEB_IDENTITY_TOKEN_FILE", + Value: "footoken", + }, + }) +} + +func testPodWithEnv(envVars []corev1.EnvVar) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foobar", + Env: envVars, + }, + }, + }, + } +} diff --git a/agent-inject/agent/container_init_sidecar.go b/agent-inject/agent/container_init_sidecar.go index 38854e60..8e7dbb6a 100644 --- a/agent-inject/agent/container_init_sidecar.go +++ b/agent-inject/agent/container_init_sidecar.go @@ -12,17 +12,40 @@ import ( // available for the agent. This means we won't need to generate // two config files. func (a *Agent) ContainerInitSidecar() (corev1.Container, error) { - volumeMounts := []corev1.VolumeMount{ - { - Name: tokenVolumeName, - MountPath: tokenVolumePath, - ReadOnly: false, - }, - { - Name: a.ServiceAccountName, - MountPath: a.ServiceAccountPath, - ReadOnly: true, - }, + volumeMounts := []corev1.VolumeMount{} + // Add aws token volume to init sidecar + if a.AwsIamTokenAccountName == "" || a.AwsIamTokenAccountPath == "" { + volumeMounts = []corev1.VolumeMount{ + { + Name: tokenVolumeName, + MountPath: tokenVolumePath, + ReadOnly: false, + }, + { + Name: a.ServiceAccountName, + MountPath: a.ServiceAccountPath, + ReadOnly: true, + }, + } + } else { + volumeMounts = []corev1.VolumeMount{ + { + Name: tokenVolumeName, + MountPath: tokenVolumePath, + ReadOnly: false, + }, + { + Name: a.ServiceAccountName, + MountPath: a.ServiceAccountPath, + ReadOnly: true, + }, + // add aws volume mounts to be available for init sidecar + { + Name: a.AwsIamTokenAccountName, + MountPath: a.AwsIamTokenAccountPath, + ReadOnly: true, + }, + } } volumeMounts = append(volumeMounts, a.ContainerVolumeMounts()...) diff --git a/agent-inject/agent/container_sidecar.go b/agent-inject/agent/container_sidecar.go index 81dbab65..33c9abd9 100644 --- a/agent-inject/agent/container_sidecar.go +++ b/agent-inject/agent/container_sidecar.go @@ -23,17 +23,39 @@ const ( // ContainerSidecar creates a new container to be added // to the pod being mutated. func (a *Agent) ContainerSidecar() (corev1.Container, error) { - volumeMounts := []corev1.VolumeMount{ - { - Name: a.ServiceAccountName, - MountPath: a.ServiceAccountPath, - ReadOnly: true, - }, - { - Name: tokenVolumeName, - MountPath: tokenVolumePath, - ReadOnly: false, - }, + volumeMounts := []corev1.VolumeMount{} + if a.AwsIamTokenAccountName == "" || a.AwsIamTokenAccountPath == "" { + volumeMounts = []corev1.VolumeMount{ + { + Name: tokenVolumeName, + MountPath: tokenVolumePath, + ReadOnly: false, + }, + { + Name: a.ServiceAccountName, + MountPath: a.ServiceAccountPath, + ReadOnly: true, + }, + } + } else { + volumeMounts = []corev1.VolumeMount{ + { + Name: tokenVolumeName, + MountPath: tokenVolumePath, + ReadOnly: false, + }, + { + Name: a.ServiceAccountName, + MountPath: a.ServiceAccountPath, + ReadOnly: true, + }, + // add aws volume mounts to be available for sidecar + { + Name: a.AwsIamTokenAccountName, + MountPath: a.AwsIamTokenAccountPath, + ReadOnly: true, + }, + } } volumeMounts = append(volumeMounts, a.ContainerVolumeMounts()...) diff --git a/agent-inject/agent/container_sidecar_test.go b/agent-inject/agent/container_sidecar_test.go index 7280d8f1..759e6fdb 100644 --- a/agent-inject/agent/container_sidecar_test.go +++ b/agent-inject/agent/container_sidecar_test.go @@ -51,16 +51,86 @@ func TestContainerSidecarVolume(t *testing.T) { require.Equal( t, []corev1.VolumeMount{ + corev1.VolumeMount{ + Name: tokenVolumeName, + MountPath: tokenVolumePath, + ReadOnly: false, + }, corev1.VolumeMount{ Name: agent.ServiceAccountName, MountPath: agent.ServiceAccountPath, ReadOnly: true, }, + corev1.VolumeMount{ + Name: secretVolumeName, + MountPath: agent.Annotations[AnnotationVaultSecretVolumePath], + ReadOnly: false, + }, + corev1.VolumeMount{ + Name: fmt.Sprintf("%s-custom-%d", secretVolumeName, 0), + MountPath: "/etc/container_environment", + ReadOnly: false, + }, + }, + container.VolumeMounts, + ) +} + +func TestContainerSidecarVolumeWithIRSA(t *testing.T) { + annotations := map[string]string{ + AnnotationVaultRole: "foobar", + // this will have different mount path + fmt.Sprintf("%s-%s", AnnotationAgentInjectSecret, "secret1"): "secrets/secret1", + fmt.Sprintf("%s-%s", AnnotationVaultSecretVolumePath, "secret1"): "/etc/container_environment", + + // this secret will have same mount path as default mount path + // adding this so we can make sure we don't have duplicate + // volume mounts + fmt.Sprintf("%s-%s", AnnotationAgentInjectSecret, "secret2"): "secret/secret2", + fmt.Sprintf("%s-%s", AnnotationVaultSecretVolumePath, "secret2"): "/etc/default_path", + + // Default path for all secrets + AnnotationVaultSecretVolumePath: "/etc/default_path", + + fmt.Sprintf("%s-%s", AnnotationAgentInjectSecret, "secret3"): "secret/secret3", + } + + pod := testPodIRSA(annotations) + var patches []*jsonpatch.JsonPatchOperation + + err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext}) + if err != nil { + t.Errorf("got error, shouldn't have: %s", err) + } + + agent, err := New(pod, patches) + if err := agent.Validate(); err != nil { + t.Errorf("agent validation failed, it shouldn't have: %s", err) + } + + container, err := agent.ContainerSidecar() + + // One token volume mount, one config volume mount and two secrets volume mounts + require.Equal(t, 5, len(container.VolumeMounts)) + + require.Equal( + t, + []corev1.VolumeMount{ corev1.VolumeMount{ Name: tokenVolumeName, MountPath: tokenVolumePath, ReadOnly: false, }, + corev1.VolumeMount{ + Name: agent.ServiceAccountName, + MountPath: agent.ServiceAccountPath, + ReadOnly: true, + }, + corev1.VolumeMount{ + Name: agent.AwsIamTokenAccountName, + MountPath: agent.AwsIamTokenAccountPath, + ReadOnly: true, + }, corev1.VolumeMount{ Name: secretVolumeName, MountPath: agent.Annotations[AnnotationVaultSecretVolumePath], From 901686058dc3d364e200834be9e76f424ccd737f Mon Sep 17 00:00:00 2001 From: Mahesh Hadimani Date: Fri, 12 Mar 2021 13:28:59 +0530 Subject: [PATCH 2/4] fix failing tests --- agent-inject/agent/container_env.go | 2 +- agent-inject/agent/container_env_test.go | 4 ++- agent-inject/agent/container_sidecar_test.go | 26 +++++++++++--------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/agent-inject/agent/container_env.go b/agent-inject/agent/container_env.go index a12f3f88..2d16e128 100644 --- a/agent-inject/agent/container_env.go +++ b/agent-inject/agent/container_env.go @@ -2,7 +2,7 @@ package agent import ( "encoding/base64" - + corev1 "k8s.io/api/core/v1" ) diff --git a/agent-inject/agent/container_env_test.go b/agent-inject/agent/container_env_test.go index c9944f5b..e570c581 100644 --- a/agent-inject/agent/container_env_test.go +++ b/agent-inject/agent/container_env_test.go @@ -45,7 +45,9 @@ func TestContainerEnvsForIRSA(t *testing.T) { expectedEnvs []string }{ {Agent{Pod: testPodWithoutIRSA()}, []string{"VAULT_CONFIG"}}, - {Agent{Pod: testPodWithIRSA()}, []string{"VAULT_CONFIG", "AWS_ROLE_ARN", "AWS_WEB_IDENTITY_TOKEN_FILE"}}, + {Agent{Pod: testPodWithIRSA(), Vault: Vault{AuthType: "aws",}}, + []string{"VAULT_CONFIG", "AWS_ROLE_ARN", "AWS_WEB_IDENTITY_TOKEN_FILE"}, + }, } for _, tt := range envTests { envs, err := tt.agent.ContainerEnvVars(true) diff --git a/agent-inject/agent/container_sidecar_test.go b/agent-inject/agent/container_sidecar_test.go index ea74ff1b..cc90cd28 100644 --- a/agent-inject/agent/container_sidecar_test.go +++ b/agent-inject/agent/container_sidecar_test.go @@ -57,16 +57,16 @@ func TestContainerSidecarVolume(t *testing.T) { require.Equal( t, []corev1.VolumeMount{ - corev1.VolumeMount{ - Name: tokenVolumeNameSidecar, - MountPath: tokenVolumePath, - ReadOnly: false, - }, corev1.VolumeMount{ Name: agent.ServiceAccountName, MountPath: agent.ServiceAccountPath, ReadOnly: true, }, + corev1.VolumeMount{ + Name: tokenVolumeNameSidecar, + MountPath: tokenVolumePath, + ReadOnly: false, + }, corev1.VolumeMount{ Name: secretVolumeName, MountPath: agent.Annotations[AnnotationVaultSecretVolumePath], @@ -93,6 +93,7 @@ func TestContainerSidecarVolume(t *testing.T) { } func TestContainerSidecarVolumeWithIRSA(t *testing.T) { + annotations := map[string]string{ AnnotationVaultRole: "foobar", // this will have different mount path @@ -114,12 +115,15 @@ func TestContainerSidecarVolumeWithIRSA(t *testing.T) { pod := testPodIRSA(annotations) var patches []*jsonpatch.JsonPatchOperation - err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext}) + err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", "aws", "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""}) if err != nil { t.Errorf("got error, shouldn't have: %s", err) } agent, err := New(pod, patches) + agent.AwsIamTokenAccountName = "aws-iam-token" + agent.AwsIamTokenAccountPath = "/var/run/secrets/eks.amazonaws.com/serviceaccount" + if err := agent.Validate(); err != nil { t.Errorf("agent validation failed, it shouldn't have: %s", err) } @@ -132,16 +136,16 @@ func TestContainerSidecarVolumeWithIRSA(t *testing.T) { require.Equal( t, []corev1.VolumeMount{ - corev1.VolumeMount{ - Name: tokenVolumeNameSidecar, - MountPath: tokenVolumePath, - ReadOnly: false, - }, corev1.VolumeMount{ Name: agent.ServiceAccountName, MountPath: agent.ServiceAccountPath, ReadOnly: true, }, + corev1.VolumeMount{ + Name: tokenVolumeNameSidecar, + MountPath: tokenVolumePath, + ReadOnly: false, + }, corev1.VolumeMount{ Name: agent.AwsIamTokenAccountName, MountPath: agent.AwsIamTokenAccountPath, From bff0bb51b251b43801897299cf6fc511a0f724be Mon Sep 17 00:00:00 2001 From: Mahesh Hadimani Date: Mon, 5 Apr 2021 12:59:58 +0530 Subject: [PATCH 3/4] removed check for length of envmap --- agent-inject/agent/container_env.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/agent-inject/agent/container_env.go b/agent-inject/agent/container_env.go index 2d16e128..b07720cf 100644 --- a/agent-inject/agent/container_env.go +++ b/agent-inject/agent/container_env.go @@ -2,7 +2,6 @@ package agent import ( "encoding/base64" - corev1 "k8s.io/api/core/v1" ) @@ -62,13 +61,11 @@ func (a *Agent) ContainerEnvVars(init bool) ([]corev1.EnvVar, error) { // Add IRSA AWS Env variables for vault containers if a.Vault.AuthType == "aws" { envMap := a.getAwsEnvsFromContainer(a.Pod) - if len(envMap) == 0 || len(envMap) >= 2 { - for k, v := range envMap { - envs = append(envs, corev1.EnvVar{ - Name: k, - Value: v, - }) - } + for k, v := range envMap { + envs = append(envs, corev1.EnvVar{ + Name: k, + Value: v, + }) } } From 0ff01f542ac3903b2099b8b8c760e06bd67ee0b5 Mon Sep 17 00:00:00 2001 From: Mahesh Hadimani Date: Tue, 13 Apr 2021 22:57:19 +0530 Subject: [PATCH 4/4] perform strict compare for env names --- agent-inject/agent/agent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent-inject/agent/agent.go b/agent-inject/agent/agent.go index 33ee0b07..9c3b4ff3 100644 --- a/agent-inject/agent/agent.go +++ b/agent-inject/agent/agent.go @@ -620,7 +620,7 @@ func (a *Agent) getAwsEnvsFromContainer(pod *corev1.Pod) map[string]string { envMap := make(map[string]string) for _, container := range pod.Spec.Containers { for _, env := range container.Env { - if strings.Contains(env.Name, "AWS_ROLE_ARN") || strings.Contains(env.Name, "AWS_WEB_IDENTITY_TOKEN_FILE") { + if env.Name == "AWS_ROLE_ARN" || env.Name == "AWS_WEB_IDENTITY_TOKEN_FILE" { if _, ok := envMap[env.Name]; !ok { envMap[env.Name] = env.Value }