Skip to content

Commit

Permalink
allow cpu/mem requests and limits to be set globally in the injector
Browse files Browse the repository at this point in the history 
Adds new flags and env vars to the injector config that allow changing the cpu/mem
requests and limits of the injected containers:

- `-cpu-request` / `AGENT_INJECT_CPU_REQUEST`
- `-cpu-limit` / `AGENT_INJECT_CPU_LIMIT`
- `-mem-request` / `AGENT_INJECT_MEM_REQUEST`
- `-mem-limit` / `AGENT_INJECT_MEM_LIMIT`
  • Loading branch information
@joemiller
joemiller committed Apr 2, 2021
1 parent 2e93af9 commit 5c045d6
Show file tree
Hide file tree
Showing 9 changed files with 122 additions and 9 deletions.
12 changes: 8 additions & 4 deletions agent-inject/agent/annotations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,10 @@ type AgentConfig struct {
SameID bool
SetSecurityContext bool
ProxyAddress string
ResourceRequestCPU string
ResourceRequestMem string
ResourceLimitCPU string
ResourceLimitMem string
}

// Init configures the expected annotations required to create a new instance
Expand Down Expand Up @@ -289,19 +293,19 @@ func Init(pod *corev1.Pod, cfg AgentConfig) error {
}

if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentLimitsCPU]; !ok {
pod.ObjectMeta.Annotations[AnnotationAgentLimitsCPU] = DefaultResourceLimitCPU
pod.ObjectMeta.Annotations[AnnotationAgentLimitsCPU] = cfg.ResourceLimitCPU
}

if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentLimitsMem]; !ok {
pod.ObjectMeta.Annotations[AnnotationAgentLimitsMem] = DefaultResourceLimitMem
pod.ObjectMeta.Annotations[AnnotationAgentLimitsMem] = cfg.ResourceLimitMem
}

if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentRequestsCPU]; !ok {
pod.ObjectMeta.Annotations[AnnotationAgentRequestsCPU] = DefaultResourceRequestCPU
pod.ObjectMeta.Annotations[AnnotationAgentRequestsCPU] = cfg.ResourceRequestCPU
}

if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentRequestsMem]; !ok {
pod.ObjectMeta.Annotations[AnnotationAgentRequestsMem] = DefaultResourceRequestMem
pod.ObjectMeta.Annotations[AnnotationAgentRequestsMem] = cfg.ResourceRequestMem
}

if _, ok := pod.ObjectMeta.Annotations[AnnotationVaultSecretVolumePath]; !ok {
Expand Down
13 changes: 13 additions & 0 deletions agent-inject/agent/annotations_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ func TestInitCanSet(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "http://proxy:3128",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -58,6 +59,7 @@ func TestInitDefaults(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "", "",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -92,6 +94,7 @@ func TestInitError(t *testing.T) {
agentConfig := AgentConfig{
"image", "", DefaultVaultAuthType, "authPath", "namespace", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err == nil {
Expand Down Expand Up @@ -156,6 +159,7 @@ func TestSecretAnnotationsWithPreserveCaseSensitivityFlagOff(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -207,6 +211,7 @@ func TestSecretAnnotationsWithPreserveCaseSensitivityFlagOn(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -294,6 +299,7 @@ func TestSecretLocationFileAnnotations(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -379,6 +385,7 @@ func TestSecretTemplateAnnotations(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -439,6 +446,7 @@ func TestTemplateShortcuts(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -499,6 +507,7 @@ func TestSecretCommandAnnotations(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -632,6 +641,7 @@ func TestCouldErrorAnnotations(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand All @@ -653,6 +663,7 @@ func TestInitEmptyPod(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err == nil {
Expand Down Expand Up @@ -682,6 +693,7 @@ func TestVaultNamespaceAnnotation(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -842,6 +854,7 @@ func TestAuthConfigAnnotations(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down
5 changes: 5 additions & 0 deletions agent-inject/agent/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ func TestNewConfig(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "http://proxy:3128",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -212,6 +213,7 @@ func TestFilePathAndName(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -244,6 +246,7 @@ func TestConfigVaultAgentCacheNotEnabledByDefault(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -283,6 +286,7 @@ func TestConfigVaultAgentCache(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -415,6 +419,7 @@ func TestConfigVaultAgentCache_persistent(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
require.NoError(t, err, "got error initialising pod: %s", err)
Expand Down
35 changes: 30 additions & 5 deletions agent-inject/agent/container_sidecar_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,12 @@ func TestContainerSidecarVolume(t *testing.T) {
pod := testPod(annotations)
var patches []*jsonpatch.JsonPatchOperation

err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""})
agentConfig := AgentConfig{
"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
t.Errorf("got error, shouldn't have: %s", err)
}
Expand Down Expand Up @@ -102,7 +107,12 @@ func TestContainerSidecar(t *testing.T) {
pod := testPod(annotations)
var patches []*jsonpatch.JsonPatchOperation

err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", false, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "http://proxy:3128"})
agentConfig := AgentConfig{
"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", false, "1000", "100",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "https://proxy:3128",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
t.Errorf("got error, shouldn't have: %s", err)
}
Expand Down Expand Up @@ -215,7 +225,12 @@ func TestContainerSidecarRevokeHook(t *testing.T) {
pod := testPod(annotations)
var patches []*jsonpatch.JsonPatchOperation

err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", tt.revokeFlag, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""})
agentConfig := AgentConfig{
"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", tt.revokeFlag, "1000", "100",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
t.Errorf("got error, shouldn't have: %s", err)
}
Expand Down Expand Up @@ -264,7 +279,12 @@ func TestContainerSidecarConfigMap(t *testing.T) {
pod := testPod(annotations)
var patches []*jsonpatch.JsonPatchOperation

err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""})
agentConfig := AgentConfig{
"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
t.Errorf("got error, shouldn't have: %s", err)
}
Expand Down Expand Up @@ -960,7 +980,12 @@ func TestContainerCache(t *testing.T) {
pod := testPod(tt.annotations)
var patches []*jsonpatch.JsonPatchOperation

err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""})
agentConfig := AgentConfig{
"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
require.NoError(t, err)

agent, err := New(pod, patches)
Expand Down
8 changes: 8 additions & 0 deletions agent-inject/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ type Handler struct {
GroupID string
SameID bool
SetSecurityContext bool
ResourceRequestCPU string
ResourceRequestMem string
ResourceLimitCPU string
ResourceLimitMem string
}

// Handle is the http.HandlerFunc implementation that actually handles the
Expand Down Expand Up @@ -154,6 +158,10 @@ func (h *Handler) Mutate(req *v1beta1.AdmissionRequest) *v1beta1.AdmissionRespon
GroupID: h.GroupID,
SameID: h.SameID,
SetSecurityContext: h.SetSecurityContext,
ResourceRequestCPU: h.ResourceRequestCPU,
ResourceRequestMem: h.ResourceRequestMem,
ResourceLimitCPU: h.ResourceLimitCPU,
ResourceLimitMem: h.ResourceLimitMem,
}
err = agent.Init(&pod, cfg)
if err != nil {
Expand Down
8 changes: 8 additions & 0 deletions deploy/injector-deployment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,14 @@ spec:
value: "vault-agent-injector-svc,vault-agent-injector-svc.$(NAMESPACE),vault-agent-injector-svc.$(NAMESPACE).svc"
- name: AGENT_INJECT_USE_LEADER_ELECTOR
value: "true"
- name: AGENT_INJECT_CPU_REQUEST
value: "250m"
- name: AGENT_INJECT_MEM_REQUEST
value: "64Mi"
- name: AGENT_INJECT_CPU_LIMIT
value: "500m"
- name: AGENT_INJECT_MEM_LIMIT
value: "128Mi"
args:
- agent-inject
- 2>&1
Expand Down
8 changes: 8 additions & 0 deletions subcommand/injector/command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ type Command struct {
flagSetSecurityContext bool // Set SecurityContext in injected containers
flagTelemetryPath string // Path under which to expose metrics
flagUseLeaderElector bool // Use leader elector code
flagResourceRequestCPU string // Set CPU request in the injected containers
flagResourceRequestMem string // Set Memory request in the injected containers
flagResourceLimitCPU string // Set CPU limit in the injected containers
flagResourceLimitMem string // Set Memory limit in the injected containers

flagSet *flag.FlagSet

Expand Down Expand Up @@ -157,6 +161,10 @@ func (c *Command) Run(args []string) int {
GroupID: c.flagRunAsGroup,
SameID: c.flagRunAsSameUser,
SetSecurityContext: c.flagSetSecurityContext,
ResourceRequestCPU: c.flagResourceRequestCPU,
ResourceRequestMem: c.flagResourceRequestMem,
ResourceLimitCPU: c.flagResourceLimitCPU,
ResourceLimitMem: c.flagResourceLimitMem,
}

mux := http.NewServeMux()
Expand Down
38 changes: 38 additions & 0 deletions subcommand/injector/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,18 @@ type Specification struct {

// UseLeaderElector is the AGENT_INJECT_USE_LEADER_ELECTOR environment variable.
UseLeaderElector string `split_words:"true"`

// ResourceRequestCPU is the AGENT_INJECT_CPU_REQUEST environment variable.
ResourceRequestCPU string `envconfig:"AGENT_INJECT_CPU_REQUEST"`

// ResourceRequestMem is the AGENT_INJECT_MEM_REQUEST environment variable.
ResourceRequestMem string `envconfig:"AGENT_INJECT_MEM_REQUEST"`

// ResourceLimitCPU is the AGENT_INJECT_CPU_LIMIT environment variable.
ResourceLimitCPU string `envconfig:"AGENT_INJECT_CPU_LIMIT"`

// ResourceLimitMem is the AGENT_INJECT_MEM_LIMIT environment variable.
ResourceLimitMem string `envconfig:"AGENT_INJECT_MEM_LIMIT"`
}

func (c *Command) init() {
Expand Down Expand Up @@ -121,6 +133,16 @@ func (c *Command) init() {
c.flagSet.BoolVar(&c.flagUseLeaderElector, "use-leader-elector", agent.DefaultAgentUseLeaderElector,
fmt.Sprintf("Use leader elector to coordinate multiple replicas when updating CA and Certs with auto-tls"))

c.flagSet.StringVar(&c.flagResourceRequestCPU, "cpu-request", agent.DefaultResourceRequestCPU,
fmt.Sprintf("CPU resource request set in injected containers. Defaults to %s", agent.DefaultResourceRequestCPU))
c.flagSet.StringVar(&c.flagResourceRequestMem, "memory-request", agent.DefaultResourceRequestMem,
fmt.Sprintf("Memory resource request set in injected containers. Defaults to %s", agent.DefaultResourceRequestMem))

c.flagSet.StringVar(&c.flagResourceLimitCPU, "cpu-limit", agent.DefaultResourceLimitCPU,
fmt.Sprintf("CPU resource limit set in injected containers. Defaults to %s", agent.DefaultResourceLimitCPU))
c.flagSet.StringVar(&c.flagResourceLimitMem, "memory-limit", agent.DefaultResourceLimitMem,
fmt.Sprintf("Memory resource limit set in injected containers. Defaults to %s", agent.DefaultResourceLimitMem))

c.help = flags.Usage(help, c.flagSet)
}

Expand Down Expand Up @@ -241,5 +263,21 @@ func (c *Command) parseEnvs() error {
}
}

if envs.ResourceRequestCPU != "" {
c.flagResourceRequestCPU = envs.ResourceRequestCPU
}

if envs.ResourceRequestMem != "" {
c.flagResourceRequestMem = envs.ResourceRequestMem
}

if envs.ResourceLimitCPU != "" {
c.flagResourceLimitCPU = envs.ResourceLimitCPU
}

if envs.ResourceLimitMem != "" {
c.flagResourceLimitMem = envs.ResourceLimitMem
}

return nil
}
4 changes: 4 additions & 0 deletions subcommand/injector/flags_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,10 @@ func TestCommandEnvs(t *testing.T) {
{env: "AGENT_INJECT_RUN_AS_USER", value: "1000", cmdPtr: &cmd.flagRunAsUser},
{env: "AGENT_INJECT_RUN_AS_GROUP", value: "1001", cmdPtr: &cmd.flagRunAsGroup},
{env: "AGENT_INJECT_TELEMETRY_PATH", value: "/metrics", cmdPtr: &cmd.flagTelemetryPath},
{env: "AGENT_INJECT_CPU_REQUEST", value: "10m", cmdPtr: &cmd.flagResourceRequestCPU},
{env: "AGENT_INJECT_MEM_REQUEST", value: "256m", cmdPtr: &cmd.flagResourceRequestMem},
{env: "AGENT_INJECT_CPU_LIMIT", value: "1000m", cmdPtr: &cmd.flagResourceLimitCPU},
{env: "AGENT_INJECT_MEM_LIMIT", value: "256m", cmdPtr: &cmd.flagResourceLimitMem},
}

for _, tt := range tests {
Expand Down

0 comments on commit 5c045d6

Please sign in to comment.