From 5c045d641c013b8b5b603306baef458517b1c00b Mon Sep 17 00:00:00 2001 From: joe miller Date: Fri, 19 Mar 2021 16:25:35 -0700 Subject: [PATCH] allow cpu/mem requests and limits to be set globally in the injector Adds new flags and env vars to the injector config that allow changing the cpu/mem requests and limits of the injected containers: - `-cpu-request` / `AGENT_INJECT_CPU_REQUEST` - `-cpu-limit` / `AGENT_INJECT_CPU_LIMIT` - `-mem-request` / `AGENT_INJECT_MEM_REQUEST` - `-mem-limit` / `AGENT_INJECT_MEM_LIMIT` --- agent-inject/agent/annotations.go | 12 ++++--- agent-inject/agent/annotations_test.go | 13 +++++++ agent-inject/agent/config_test.go | 5 +++ agent-inject/agent/container_sidecar_test.go | 35 +++++++++++++++--- agent-inject/handler.go | 8 +++++ deploy/injector-deployment.yaml | 8 +++++ subcommand/injector/command.go | 8 +++++ subcommand/injector/flags.go | 38 ++++++++++++++++++++ subcommand/injector/flags_test.go | 4 +++ 9 files changed, 122 insertions(+), 9 deletions(-) diff --git a/agent-inject/agent/annotations.go b/agent-inject/agent/annotations.go index d0b1f4eb..b9e07e32 100644 --- a/agent-inject/agent/annotations.go +++ b/agent-inject/agent/annotations.go @@ -227,6 +227,10 @@ type AgentConfig struct { SameID bool SetSecurityContext bool ProxyAddress string + ResourceRequestCPU string + ResourceRequestMem string + ResourceLimitCPU string + ResourceLimitMem string } // Init configures the expected annotations required to create a new instance @@ -289,19 +293,19 @@ func Init(pod *corev1.Pod, cfg AgentConfig) error { } if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentLimitsCPU]; !ok { - pod.ObjectMeta.Annotations[AnnotationAgentLimitsCPU] = DefaultResourceLimitCPU + pod.ObjectMeta.Annotations[AnnotationAgentLimitsCPU] = cfg.ResourceLimitCPU } if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentLimitsMem]; !ok { - pod.ObjectMeta.Annotations[AnnotationAgentLimitsMem] = DefaultResourceLimitMem + pod.ObjectMeta.Annotations[AnnotationAgentLimitsMem] = cfg.ResourceLimitMem } if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentRequestsCPU]; !ok { - pod.ObjectMeta.Annotations[AnnotationAgentRequestsCPU] = DefaultResourceRequestCPU + pod.ObjectMeta.Annotations[AnnotationAgentRequestsCPU] = cfg.ResourceRequestCPU } if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentRequestsMem]; !ok { - pod.ObjectMeta.Annotations[AnnotationAgentRequestsMem] = DefaultResourceRequestMem + pod.ObjectMeta.Annotations[AnnotationAgentRequestsMem] = cfg.ResourceRequestMem } if _, ok := pod.ObjectMeta.Annotations[AnnotationVaultSecretVolumePath]; !ok { diff --git a/agent-inject/agent/annotations_test.go b/agent-inject/agent/annotations_test.go index 8467e519..bb779ac2 100644 --- a/agent-inject/agent/annotations_test.go +++ b/agent-inject/agent/annotations_test.go @@ -21,6 +21,7 @@ func TestInitCanSet(t *testing.T) { agentConfig := AgentConfig{ "foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "http://proxy:3128", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -58,6 +59,7 @@ func TestInitDefaults(t *testing.T) { agentConfig := AgentConfig{ "", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "", "", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -92,6 +94,7 @@ func TestInitError(t *testing.T) { agentConfig := AgentConfig{ "image", "", DefaultVaultAuthType, "authPath", "namespace", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err == nil { @@ -156,6 +159,7 @@ func TestSecretAnnotationsWithPreserveCaseSensitivityFlagOff(t *testing.T) { agentConfig := AgentConfig{ "", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -207,6 +211,7 @@ func TestSecretAnnotationsWithPreserveCaseSensitivityFlagOn(t *testing.T) { agentConfig := AgentConfig{ "", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -294,6 +299,7 @@ func TestSecretLocationFileAnnotations(t *testing.T) { agentConfig := AgentConfig{ "", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -379,6 +385,7 @@ func TestSecretTemplateAnnotations(t *testing.T) { agentConfig := AgentConfig{ "", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -439,6 +446,7 @@ func TestTemplateShortcuts(t *testing.T) { agentConfig := AgentConfig{ "", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -499,6 +507,7 @@ func TestSecretCommandAnnotations(t *testing.T) { agentConfig := AgentConfig{ "", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -632,6 +641,7 @@ func TestCouldErrorAnnotations(t *testing.T) { agentConfig := AgentConfig{ "", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -653,6 +663,7 @@ func TestInitEmptyPod(t *testing.T) { agentConfig := AgentConfig{ "foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err == nil { @@ -682,6 +693,7 @@ func TestVaultNamespaceAnnotation(t *testing.T) { agentConfig := AgentConfig{ "foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -842,6 +854,7 @@ func TestAuthConfigAnnotations(t *testing.T) { agentConfig := AgentConfig{ "", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { diff --git a/agent-inject/agent/config_test.go b/agent-inject/agent/config_test.go index cf5e4504..a4ed187c 100644 --- a/agent-inject/agent/config_test.go +++ b/agent-inject/agent/config_test.go @@ -46,6 +46,7 @@ func TestNewConfig(t *testing.T) { agentConfig := AgentConfig{ "foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "http://proxy:3128", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -212,6 +213,7 @@ func TestFilePathAndName(t *testing.T) { agentConfig := AgentConfig{ "foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -244,6 +246,7 @@ func TestConfigVaultAgentCacheNotEnabledByDefault(t *testing.T) { agentConfig := AgentConfig{ "foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -283,6 +286,7 @@ func TestConfigVaultAgentCache(t *testing.T) { agentConfig := AgentConfig{ "foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) if err != nil { @@ -415,6 +419,7 @@ func TestConfigVaultAgentCache_persistent(t *testing.T) { agentConfig := AgentConfig{ "foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, } err := Init(pod, agentConfig) require.NoError(t, err, "got error initialising pod: %s", err) diff --git a/agent-inject/agent/container_sidecar_test.go b/agent-inject/agent/container_sidecar_test.go index a62da839..78030968 100644 --- a/agent-inject/agent/container_sidecar_test.go +++ b/agent-inject/agent/container_sidecar_test.go @@ -41,7 +41,12 @@ func TestContainerSidecarVolume(t *testing.T) { pod := testPod(annotations) var patches []*jsonpatch.JsonPatchOperation - err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""}) + agentConfig := AgentConfig{ + "foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", + DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, + } + err := Init(pod, agentConfig) if err != nil { t.Errorf("got error, shouldn't have: %s", err) } @@ -102,7 +107,12 @@ func TestContainerSidecar(t *testing.T) { pod := testPod(annotations) var patches []*jsonpatch.JsonPatchOperation - err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", false, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "http://proxy:3128"}) + agentConfig := AgentConfig{ + "foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", false, "1000", "100", + DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "https://proxy:3128", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, + } + err := Init(pod, agentConfig) if err != nil { t.Errorf("got error, shouldn't have: %s", err) } @@ -215,7 +225,12 @@ func TestContainerSidecarRevokeHook(t *testing.T) { pod := testPod(annotations) var patches []*jsonpatch.JsonPatchOperation - err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", tt.revokeFlag, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""}) + agentConfig := AgentConfig{ + "foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", tt.revokeFlag, "1000", "100", + DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, + } + err := Init(pod, agentConfig) if err != nil { t.Errorf("got error, shouldn't have: %s", err) } @@ -264,7 +279,12 @@ func TestContainerSidecarConfigMap(t *testing.T) { pod := testPod(annotations) var patches []*jsonpatch.JsonPatchOperation - err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""}) + agentConfig := AgentConfig{ + "foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", + DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, + } + err := Init(pod, agentConfig) if err != nil { t.Errorf("got error, shouldn't have: %s", err) } @@ -960,7 +980,12 @@ func TestContainerCache(t *testing.T) { pod := testPod(tt.annotations) var patches []*jsonpatch.JsonPatchOperation - err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""}) + agentConfig := AgentConfig{ + "foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", + DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "", + DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem, + } + err := Init(pod, agentConfig) require.NoError(t, err) agent, err := New(pod, patches) diff --git a/agent-inject/handler.go b/agent-inject/handler.go index e0361e5f..571c1b8d 100644 --- a/agent-inject/handler.go +++ b/agent-inject/handler.go @@ -48,6 +48,10 @@ type Handler struct { GroupID string SameID bool SetSecurityContext bool + ResourceRequestCPU string + ResourceRequestMem string + ResourceLimitCPU string + ResourceLimitMem string } // Handle is the http.HandlerFunc implementation that actually handles the @@ -154,6 +158,10 @@ func (h *Handler) Mutate(req *v1beta1.AdmissionRequest) *v1beta1.AdmissionRespon GroupID: h.GroupID, SameID: h.SameID, SetSecurityContext: h.SetSecurityContext, + ResourceRequestCPU: h.ResourceRequestCPU, + ResourceRequestMem: h.ResourceRequestMem, + ResourceLimitCPU: h.ResourceLimitCPU, + ResourceLimitMem: h.ResourceLimitMem, } err = agent.Init(&pod, cfg) if err != nil { diff --git a/deploy/injector-deployment.yaml b/deploy/injector-deployment.yaml index b55ae4a5..22a3712a 100644 --- a/deploy/injector-deployment.yaml +++ b/deploy/injector-deployment.yaml @@ -76,6 +76,14 @@ spec: value: "vault-agent-injector-svc,vault-agent-injector-svc.$(NAMESPACE),vault-agent-injector-svc.$(NAMESPACE).svc" - name: AGENT_INJECT_USE_LEADER_ELECTOR value: "true" + - name: AGENT_INJECT_CPU_REQUEST + value: "250m" + - name: AGENT_INJECT_MEM_REQUEST + value: "64Mi" + - name: AGENT_INJECT_CPU_LIMIT + value: "500m" + - name: AGENT_INJECT_MEM_LIMIT + value: "128Mi" args: - agent-inject - 2>&1 diff --git a/subcommand/injector/command.go b/subcommand/injector/command.go index 5418d0be..49971da8 100644 --- a/subcommand/injector/command.go +++ b/subcommand/injector/command.go @@ -51,6 +51,10 @@ type Command struct { flagSetSecurityContext bool // Set SecurityContext in injected containers flagTelemetryPath string // Path under which to expose metrics flagUseLeaderElector bool // Use leader elector code + flagResourceRequestCPU string // Set CPU request in the injected containers + flagResourceRequestMem string // Set Memory request in the injected containers + flagResourceLimitCPU string // Set CPU limit in the injected containers + flagResourceLimitMem string // Set Memory limit in the injected containers flagSet *flag.FlagSet @@ -157,6 +161,10 @@ func (c *Command) Run(args []string) int { GroupID: c.flagRunAsGroup, SameID: c.flagRunAsSameUser, SetSecurityContext: c.flagSetSecurityContext, + ResourceRequestCPU: c.flagResourceRequestCPU, + ResourceRequestMem: c.flagResourceRequestMem, + ResourceLimitCPU: c.flagResourceLimitCPU, + ResourceLimitMem: c.flagResourceLimitMem, } mux := http.NewServeMux() diff --git a/subcommand/injector/flags.go b/subcommand/injector/flags.go index 7411afab..431e8ba3 100644 --- a/subcommand/injector/flags.go +++ b/subcommand/injector/flags.go @@ -77,6 +77,18 @@ type Specification struct { // UseLeaderElector is the AGENT_INJECT_USE_LEADER_ELECTOR environment variable. UseLeaderElector string `split_words:"true"` + + // ResourceRequestCPU is the AGENT_INJECT_CPU_REQUEST environment variable. + ResourceRequestCPU string `envconfig:"AGENT_INJECT_CPU_REQUEST"` + + // ResourceRequestMem is the AGENT_INJECT_MEM_REQUEST environment variable. + ResourceRequestMem string `envconfig:"AGENT_INJECT_MEM_REQUEST"` + + // ResourceLimitCPU is the AGENT_INJECT_CPU_LIMIT environment variable. + ResourceLimitCPU string `envconfig:"AGENT_INJECT_CPU_LIMIT"` + + // ResourceLimitMem is the AGENT_INJECT_MEM_LIMIT environment variable. + ResourceLimitMem string `envconfig:"AGENT_INJECT_MEM_LIMIT"` } func (c *Command) init() { @@ -121,6 +133,16 @@ func (c *Command) init() { c.flagSet.BoolVar(&c.flagUseLeaderElector, "use-leader-elector", agent.DefaultAgentUseLeaderElector, fmt.Sprintf("Use leader elector to coordinate multiple replicas when updating CA and Certs with auto-tls")) + c.flagSet.StringVar(&c.flagResourceRequestCPU, "cpu-request", agent.DefaultResourceRequestCPU, + fmt.Sprintf("CPU resource request set in injected containers. Defaults to %s", agent.DefaultResourceRequestCPU)) + c.flagSet.StringVar(&c.flagResourceRequestMem, "memory-request", agent.DefaultResourceRequestMem, + fmt.Sprintf("Memory resource request set in injected containers. Defaults to %s", agent.DefaultResourceRequestMem)) + + c.flagSet.StringVar(&c.flagResourceLimitCPU, "cpu-limit", agent.DefaultResourceLimitCPU, + fmt.Sprintf("CPU resource limit set in injected containers. Defaults to %s", agent.DefaultResourceLimitCPU)) + c.flagSet.StringVar(&c.flagResourceLimitMem, "memory-limit", agent.DefaultResourceLimitMem, + fmt.Sprintf("Memory resource limit set in injected containers. Defaults to %s", agent.DefaultResourceLimitMem)) + c.help = flags.Usage(help, c.flagSet) } @@ -241,5 +263,21 @@ func (c *Command) parseEnvs() error { } } + if envs.ResourceRequestCPU != "" { + c.flagResourceRequestCPU = envs.ResourceRequestCPU + } + + if envs.ResourceRequestMem != "" { + c.flagResourceRequestMem = envs.ResourceRequestMem + } + + if envs.ResourceLimitCPU != "" { + c.flagResourceLimitCPU = envs.ResourceLimitCPU + } + + if envs.ResourceLimitMem != "" { + c.flagResourceLimitMem = envs.ResourceLimitMem + } + return nil } diff --git a/subcommand/injector/flags_test.go b/subcommand/injector/flags_test.go index 6645d4a4..df9eed99 100644 --- a/subcommand/injector/flags_test.go +++ b/subcommand/injector/flags_test.go @@ -125,6 +125,10 @@ func TestCommandEnvs(t *testing.T) { {env: "AGENT_INJECT_RUN_AS_USER", value: "1000", cmdPtr: &cmd.flagRunAsUser}, {env: "AGENT_INJECT_RUN_AS_GROUP", value: "1001", cmdPtr: &cmd.flagRunAsGroup}, {env: "AGENT_INJECT_TELEMETRY_PATH", value: "/metrics", cmdPtr: &cmd.flagTelemetryPath}, + {env: "AGENT_INJECT_CPU_REQUEST", value: "10m", cmdPtr: &cmd.flagResourceRequestCPU}, + {env: "AGENT_INJECT_MEM_REQUEST", value: "256m", cmdPtr: &cmd.flagResourceRequestMem}, + {env: "AGENT_INJECT_CPU_LIMIT", value: "1000m", cmdPtr: &cmd.flagResourceLimitCPU}, + {env: "AGENT_INJECT_MEM_LIMIT", value: "256m", cmdPtr: &cmd.flagResourceLimitMem}, } for _, tt := range tests {