Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

allow cpu/mem requests and limits to be set globally in the injector #235

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions agent-inject/agent/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,10 @@ type AgentConfig struct {
SameID bool
SetSecurityContext bool
ProxyAddress string
ResourceRequestCPU string
ResourceRequestMem string
ResourceLimitCPU string
ResourceLimitMem string
}

// Init configures the expected annotations required to create a new instance
Expand Down Expand Up @@ -289,19 +293,19 @@ func Init(pod *corev1.Pod, cfg AgentConfig) error {
}

if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentLimitsCPU]; !ok {
pod.ObjectMeta.Annotations[AnnotationAgentLimitsCPU] = DefaultResourceLimitCPU
pod.ObjectMeta.Annotations[AnnotationAgentLimitsCPU] = cfg.ResourceLimitCPU
}

if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentLimitsMem]; !ok {
pod.ObjectMeta.Annotations[AnnotationAgentLimitsMem] = DefaultResourceLimitMem
pod.ObjectMeta.Annotations[AnnotationAgentLimitsMem] = cfg.ResourceLimitMem
}

if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentRequestsCPU]; !ok {
pod.ObjectMeta.Annotations[AnnotationAgentRequestsCPU] = DefaultResourceRequestCPU
pod.ObjectMeta.Annotations[AnnotationAgentRequestsCPU] = cfg.ResourceRequestCPU
}

if _, ok := pod.ObjectMeta.Annotations[AnnotationAgentRequestsMem]; !ok {
pod.ObjectMeta.Annotations[AnnotationAgentRequestsMem] = DefaultResourceRequestMem
pod.ObjectMeta.Annotations[AnnotationAgentRequestsMem] = cfg.ResourceRequestMem
}

if _, ok := pod.ObjectMeta.Annotations[AnnotationVaultSecretVolumePath]; !ok {
Expand Down
13 changes: 13 additions & 0 deletions agent-inject/agent/annotations_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ func TestInitCanSet(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "http://proxy:3128",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -58,6 +59,7 @@ func TestInitDefaults(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "", "",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -92,6 +94,7 @@ func TestInitError(t *testing.T) {
agentConfig := AgentConfig{
"image", "", DefaultVaultAuthType, "authPath", "namespace", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err == nil {
Expand Down Expand Up @@ -156,6 +159,7 @@ func TestSecretAnnotationsWithPreserveCaseSensitivityFlagOff(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -207,6 +211,7 @@ func TestSecretAnnotationsWithPreserveCaseSensitivityFlagOn(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -294,6 +299,7 @@ func TestSecretLocationFileAnnotations(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -379,6 +385,7 @@ func TestSecretTemplateAnnotations(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -439,6 +446,7 @@ func TestTemplateShortcuts(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -499,6 +507,7 @@ func TestSecretCommandAnnotations(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -632,6 +641,7 @@ func TestCouldErrorAnnotations(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand All @@ -653,6 +663,7 @@ func TestInitEmptyPod(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err == nil {
Expand Down Expand Up @@ -682,6 +693,7 @@ func TestVaultNamespaceAnnotation(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -842,6 +854,7 @@ func TestAuthConfigAnnotations(t *testing.T) {
agentConfig := AgentConfig{
"", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down
5 changes: 5 additions & 0 deletions agent-inject/agent/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ func TestNewConfig(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "http://proxy:3128",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -212,6 +213,7 @@ func TestFilePathAndName(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -244,6 +246,7 @@ func TestConfigVaultAgentCacheNotEnabledByDefault(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -283,6 +286,7 @@ func TestConfigVaultAgentCache(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
Expand Down Expand Up @@ -415,6 +419,7 @@ func TestConfigVaultAgentCache_persistent(t *testing.T) {
agentConfig := AgentConfig{
"foobar-image", "http://foobar:8200", DefaultVaultAuthType, "test", "test", true, "100", "1000",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
require.NoError(t, err, "got error initialising pod: %s", err)
Expand Down
35 changes: 30 additions & 5 deletions agent-inject/agent/container_sidecar_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,12 @@ func TestContainerSidecarVolume(t *testing.T) {
pod := testPod(annotations)
var patches []*jsonpatch.JsonPatchOperation

err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""})
agentConfig := AgentConfig{
"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
t.Errorf("got error, shouldn't have: %s", err)
}
Expand Down Expand Up @@ -102,7 +107,12 @@ func TestContainerSidecar(t *testing.T) {
pod := testPod(annotations)
var patches []*jsonpatch.JsonPatchOperation

err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", false, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "http://proxy:3128"})
agentConfig := AgentConfig{
"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", false, "1000", "100",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "https://proxy:3128",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
t.Errorf("got error, shouldn't have: %s", err)
}
Expand Down Expand Up @@ -215,7 +225,12 @@ func TestContainerSidecarRevokeHook(t *testing.T) {
pod := testPod(annotations)
var patches []*jsonpatch.JsonPatchOperation

err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", tt.revokeFlag, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""})
agentConfig := AgentConfig{
"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", tt.revokeFlag, "1000", "100",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
t.Errorf("got error, shouldn't have: %s", err)
}
Expand Down Expand Up @@ -264,7 +279,12 @@ func TestContainerSidecarConfigMap(t *testing.T) {
pod := testPod(annotations)
var patches []*jsonpatch.JsonPatchOperation

err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""})
agentConfig := AgentConfig{
"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
if err != nil {
t.Errorf("got error, shouldn't have: %s", err)
}
Expand Down Expand Up @@ -960,7 +980,12 @@ func TestContainerCache(t *testing.T) {
pod := testPod(tt.annotations)
var patches []*jsonpatch.JsonPatchOperation

err := Init(pod, AgentConfig{"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100", DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, ""})
agentConfig := AgentConfig{
"foobar-image", "http://foobar:1234", DefaultVaultAuthType, "test", "test", true, "1000", "100",
DefaultAgentRunAsSameUser, DefaultAgentSetSecurityContext, "",
DefaultResourceRequestCPU, DefaultResourceRequestMem, DefaultResourceLimitCPU, DefaultResourceLimitMem,
}
err := Init(pod, agentConfig)
require.NoError(t, err)

agent, err := New(pod, patches)
Expand Down
8 changes: 8 additions & 0 deletions agent-inject/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ type Handler struct {
GroupID string
SameID bool
SetSecurityContext bool
ResourceRequestCPU string
ResourceRequestMem string
ResourceLimitCPU string
ResourceLimitMem string
}

// Handle is the http.HandlerFunc implementation that actually handles the
Expand Down Expand Up @@ -154,6 +158,10 @@ func (h *Handler) Mutate(req *v1beta1.AdmissionRequest) *v1beta1.AdmissionRespon
GroupID: h.GroupID,
SameID: h.SameID,
SetSecurityContext: h.SetSecurityContext,
ResourceRequestCPU: h.ResourceRequestCPU,
ResourceRequestMem: h.ResourceRequestMem,
ResourceLimitCPU: h.ResourceLimitCPU,
ResourceLimitMem: h.ResourceLimitMem,
}
err = agent.Init(&pod, cfg)
if err != nil {
Expand Down
8 changes: 8 additions & 0 deletions deploy/injector-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,14 @@ spec:
value: "vault-agent-injector-svc,vault-agent-injector-svc.$(NAMESPACE),vault-agent-injector-svc.$(NAMESPACE).svc"
- name: AGENT_INJECT_USE_LEADER_ELECTOR
value: "true"
- name: AGENT_INJECT_CPU_REQUEST
jasonodonnell marked this conversation as resolved.
Show resolved Hide resolved
value: "250m"
- name: AGENT_INJECT_MEM_REQUEST
value: "64Mi"
- name: AGENT_INJECT_CPU_LIMIT
value: "500m"
- name: AGENT_INJECT_MEM_LIMIT
value: "128Mi"
args:
- agent-inject
- 2>&1
Expand Down
8 changes: 8 additions & 0 deletions subcommand/injector/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ type Command struct {
flagSetSecurityContext bool // Set SecurityContext in injected containers
flagTelemetryPath string // Path under which to expose metrics
flagUseLeaderElector bool // Use leader elector code
flagResourceRequestCPU string // Set CPU request in the injected containers
flagResourceRequestMem string // Set Memory request in the injected containers
flagResourceLimitCPU string // Set CPU limit in the injected containers
flagResourceLimitMem string // Set Memory limit in the injected containers

flagSet *flag.FlagSet

Expand Down Expand Up @@ -157,6 +161,10 @@ func (c *Command) Run(args []string) int {
GroupID: c.flagRunAsGroup,
SameID: c.flagRunAsSameUser,
SetSecurityContext: c.flagSetSecurityContext,
ResourceRequestCPU: c.flagResourceRequestCPU,
ResourceRequestMem: c.flagResourceRequestMem,
ResourceLimitCPU: c.flagResourceLimitCPU,
ResourceLimitMem: c.flagResourceLimitMem,
}

mux := http.NewServeMux()
Expand Down
38 changes: 38 additions & 0 deletions subcommand/injector/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,18 @@ type Specification struct {

// UseLeaderElector is the AGENT_INJECT_USE_LEADER_ELECTOR environment variable.
UseLeaderElector string `split_words:"true"`

// ResourceRequestCPU is the AGENT_INJECT_CPU_REQUEST environment variable.
ResourceRequestCPU string `envconfig:"AGENT_INJECT_CPU_REQUEST"`

// ResourceRequestMem is the AGENT_INJECT_MEM_REQUEST environment variable.
ResourceRequestMem string `envconfig:"AGENT_INJECT_MEM_REQUEST"`

// ResourceLimitCPU is the AGENT_INJECT_CPU_LIMIT environment variable.
ResourceLimitCPU string `envconfig:"AGENT_INJECT_CPU_LIMIT"`

// ResourceLimitMem is the AGENT_INJECT_MEM_LIMIT environment variable.
ResourceLimitMem string `envconfig:"AGENT_INJECT_MEM_LIMIT"`
}

func (c *Command) init() {
Expand Down Expand Up @@ -121,6 +133,16 @@ func (c *Command) init() {
c.flagSet.BoolVar(&c.flagUseLeaderElector, "use-leader-elector", agent.DefaultAgentUseLeaderElector,
fmt.Sprintf("Use leader elector to coordinate multiple replicas when updating CA and Certs with auto-tls"))

c.flagSet.StringVar(&c.flagResourceRequestCPU, "cpu-request", agent.DefaultResourceRequestCPU,
fmt.Sprintf("CPU resource request set in injected containers. Defaults to %s", agent.DefaultResourceRequestCPU))
c.flagSet.StringVar(&c.flagResourceRequestMem, "memory-request", agent.DefaultResourceRequestMem,
fmt.Sprintf("Memory resource request set in injected containers. Defaults to %s", agent.DefaultResourceRequestMem))

c.flagSet.StringVar(&c.flagResourceLimitCPU, "cpu-limit", agent.DefaultResourceLimitCPU,
fmt.Sprintf("CPU resource limit set in injected containers. Defaults to %s", agent.DefaultResourceLimitCPU))
c.flagSet.StringVar(&c.flagResourceLimitMem, "memory-limit", agent.DefaultResourceLimitMem,
fmt.Sprintf("Memory resource limit set in injected containers. Defaults to %s", agent.DefaultResourceLimitMem))

c.help = flags.Usage(help, c.flagSet)
}

Expand Down Expand Up @@ -241,5 +263,21 @@ func (c *Command) parseEnvs() error {
}
}

if envs.ResourceRequestCPU != "" {
c.flagResourceRequestCPU = envs.ResourceRequestCPU
}

if envs.ResourceRequestMem != "" {
c.flagResourceRequestMem = envs.ResourceRequestMem
}

if envs.ResourceLimitCPU != "" {
c.flagResourceLimitCPU = envs.ResourceLimitCPU
}

if envs.ResourceLimitMem != "" {
c.flagResourceLimitMem = envs.ResourceLimitMem
}

return nil
}
4 changes: 4 additions & 0 deletions subcommand/injector/flags_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,10 @@ func TestCommandEnvs(t *testing.T) {
{env: "AGENT_INJECT_RUN_AS_USER", value: "1000", cmdPtr: &cmd.flagRunAsUser},
{env: "AGENT_INJECT_RUN_AS_GROUP", value: "1001", cmdPtr: &cmd.flagRunAsGroup},
{env: "AGENT_INJECT_TELEMETRY_PATH", value: "/metrics", cmdPtr: &cmd.flagTelemetryPath},
{env: "AGENT_INJECT_CPU_REQUEST", value: "10m", cmdPtr: &cmd.flagResourceRequestCPU},
{env: "AGENT_INJECT_MEM_REQUEST", value: "256m", cmdPtr: &cmd.flagResourceRequestMem},
{env: "AGENT_INJECT_CPU_LIMIT", value: "1000m", cmdPtr: &cmd.flagResourceLimitCPU},
{env: "AGENT_INJECT_MEM_LIMIT", value: "256m", cmdPtr: &cmd.flagResourceLimitMem},
}

for _, tt := range tests {
Expand Down