diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..a9693892f2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +_output/ +.idea/ diff --git a/README.md b/README.md index 61d7a3d0ee..cd9194a552 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,23 @@ strategies: - "requiredDuringSchedulingIgnoredDuringExecution" ``` + +### RemovePodsHavingTooManyRestarts + +This strategy makes sure that pods having too many restarts are removed from nodes. For example a pod with EBS/PD that can't get the volume/disk attached to the instance, then the pod should be re-scheduled to other nodes. + +``` +apiVersion: "descheduler/v1alpha1" +kind: "DeschedulerPolicy" +strategies: + "RemovePodsHavingTooManyRestarts": + enabled: true + params: + podsHavingTooManyRestarts: + podeRestartThresholds: 100 + includingInitContainers: true +``` + ## Pod Evictions When the descheduler decides to evict pods from a node, it employs following general mechanism: diff --git a/examples/policy.yaml b/examples/policy.yaml index 38fd0d8f32..3f64242aa1 100644 --- a/examples/policy.yaml +++ b/examples/policy.yaml @@ -17,3 +17,9 @@ strategies: "cpu" : 50 "memory": 50 "pods": 50 + "RemovePodsHavingTooManyRestarts": + enabled: true + params: + podsHavingTooManyRestarts: + podeRestartThresholds: 100 + includingInitContainers: true diff --git a/pkg/api/types.generated.go b/pkg/api/types.generated.go index ba0b0a5cb8..d074fb1da9 100644 --- a/pkg/api/types.generated.go +++ b/pkg/api/types.generated.go @@ -673,9 +673,9 @@ func (x *StrategyParameters) CodecEncodeSelf(e *codec1978.Encoder) { _, _ = yysep2, yy2arr2 const yyr2 bool = false if yyr2 || yy2arr2 { - r.WriteArrayStart(2) + r.WriteArrayStart(3) } else { - r.WriteMapStart(2) + r.WriteMapStart(3) } if yyr2 || yy2arr2 { r.WriteArrayElem() @@ -715,6 +715,17 @@ func (x *StrategyParameters) CodecEncodeSelf(e *codec1978.Encoder) { } } } + if yyr2 || yy2arr2 { + r.WriteArrayElem() + yy12 := &x.PodsHavingTooManyRestarts + yy12.CodecEncodeSelf(e) + } else { + r.WriteMapElemKey() + r.EncStructFieldKey(codecSelferValueTypeString1234, `PodsHavingTooManyRestarts`) + r.WriteMapElemValue() + yy14 := &x.PodsHavingTooManyRestarts + yy14.CodecEncodeSelf(e) + } if yyr2 || yy2arr2 { r.WriteArrayEnd() } else { @@ -793,6 +804,13 @@ func (x *StrategyParameters) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) z.F.DecSliceStringX(yyv5, d) } } + case "PodsHavingTooManyRestarts": + if r.TryDecodeAsNil() { + x.PodsHavingTooManyRestarts = PodsHavingTooManyRestarts{} + } else { + yyv7 := &x.PodsHavingTooManyRestarts + yyv7.CodecDecodeSelf(d) + } default: z.DecStructFieldNotFound(-1, yys3) } // end switch yys3 @@ -804,16 +822,16 @@ func (x *StrategyParameters) codecDecodeSelfFromArray(l int, d *codec1978.Decode var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj7 int - var yyb7 bool - var yyhl7 bool = l >= 0 - yyj7++ - if yyhl7 { - yyb7 = yyj7 > l + var yyj8 int + var yyb8 bool + var yyhl8 bool = l >= 0 + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l } else { - yyb7 = r.CheckBreak() + yyb8 = r.CheckBreak() } - if yyb7 { + if yyb8 { r.ReadArrayEnd() return } @@ -821,16 +839,16 @@ func (x *StrategyParameters) codecDecodeSelfFromArray(l int, d *codec1978.Decode if r.TryDecodeAsNil() { x.NodeResourceUtilizationThresholds = NodeResourceUtilizationThresholds{} } else { - yyv8 := &x.NodeResourceUtilizationThresholds - yyv8.CodecDecodeSelf(d) + yyv9 := &x.NodeResourceUtilizationThresholds + yyv9.CodecDecodeSelf(d) } - yyj7++ - if yyhl7 { - yyb7 = yyj7 > l + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l } else { - yyb7 = r.CheckBreak() + yyb8 = r.CheckBreak() } - if yyb7 { + if yyb8 { r.ReadArrayEnd() return } @@ -838,26 +856,43 @@ func (x *StrategyParameters) codecDecodeSelfFromArray(l int, d *codec1978.Decode if r.TryDecodeAsNil() { x.NodeAffinityType = nil } else { - yyv9 := &x.NodeAffinityType - yym10 := z.DecBinary() - _ = yym10 + yyv10 := &x.NodeAffinityType + yym11 := z.DecBinary() + _ = yym11 if false { } else { - z.F.DecSliceStringX(yyv9, d) + z.F.DecSliceStringX(yyv10, d) } } + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + r.ReadArrayEnd() + return + } + r.ReadArrayElem() + if r.TryDecodeAsNil() { + x.PodsHavingTooManyRestarts = PodsHavingTooManyRestarts{} + } else { + yyv12 := &x.PodsHavingTooManyRestarts + yyv12.CodecDecodeSelf(d) + } for { - yyj7++ - if yyhl7 { - yyb7 = yyj7 > l + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l } else { - yyb7 = r.CheckBreak() + yyb8 = r.CheckBreak() } - if yyb7 { + if yyb8 { break } r.ReadArrayElem() - z.DecStructFieldNotFound(yyj7-1, "") + z.DecStructFieldNotFound(yyj8-1, "") } r.ReadArrayEnd() } @@ -1168,6 +1203,223 @@ func (x *NodeResourceUtilizationThresholds) codecDecodeSelfFromArray(l int, d *c r.ReadArrayEnd() } +func (x *PodsHavingTooManyRestarts) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + if x == nil { + r.EncodeNil() + } else { + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if yyxt1 := z.Extension(z.I2Rtid(x)); yyxt1 != nil { + z.EncExtension(x, yyxt1) + } else { + yysep2 := !z.EncBinary() + yy2arr2 := z.EncBasicHandle().StructToArray + _, _ = yysep2, yy2arr2 + const yyr2 bool = false + if yyr2 || yy2arr2 { + r.WriteArrayStart(2) + } else { + r.WriteMapStart(2) + } + if yyr2 || yy2arr2 { + r.WriteArrayElem() + yym4 := z.EncBinary() + _ = yym4 + if false { + } else { + r.EncodeInt(int64(x.PodeRestartThresholds)) + } + } else { + r.WriteMapElemKey() + r.EncStructFieldKey(codecSelferValueTypeString1234, `PodeRestartThresholds`) + r.WriteMapElemValue() + yym5 := z.EncBinary() + _ = yym5 + if false { + } else { + r.EncodeInt(int64(x.PodeRestartThresholds)) + } + } + if yyr2 || yy2arr2 { + r.WriteArrayElem() + yym7 := z.EncBinary() + _ = yym7 + if false { + } else { + r.EncodeBool(bool(x.IncludingInitContainers)) + } + } else { + r.WriteMapElemKey() + r.EncStructFieldKey(codecSelferValueTypeString1234, `IncludingInitContainers`) + r.WriteMapElemValue() + yym8 := z.EncBinary() + _ = yym8 + if false { + } else { + r.EncodeBool(bool(x.IncludingInitContainers)) + } + } + if yyr2 || yy2arr2 { + r.WriteArrayEnd() + } else { + r.WriteMapEnd() + } + } + } +} + +func (x *PodsHavingTooManyRestarts) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if yyxt1 := z.Extension(z.I2Rtid(x)); yyxt1 != nil { + z.DecExtension(x, yyxt1) + } else { + yyct2 := r.ContainerType() + if yyct2 == codecSelferValueTypeMap1234 { + yyl2 := r.ReadMapStart() + if yyl2 == 0 { + r.ReadMapEnd() + } else { + x.codecDecodeSelfFromMap(yyl2, d) + } + } else if yyct2 == codecSelferValueTypeArray1234 { + yyl2 := r.ReadArrayStart() + if yyl2 == 0 { + r.ReadArrayEnd() + } else { + x.codecDecodeSelfFromArray(yyl2, d) + } + } else { + panic(errCodecSelferOnlyMapOrArrayEncodeToStruct1234) + } + } +} + +func (x *PodsHavingTooManyRestarts) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyhl3 bool = l >= 0 + for yyj3 := 0; ; yyj3++ { + if yyhl3 { + if yyj3 >= l { + break + } + } else { + if r.CheckBreak() { + break + } + } + r.ReadMapElemKey() + yys3 := z.StringView(r.DecStructFieldKey(codecSelferValueTypeString1234, z.DecScratchArrayBuffer())) + r.ReadMapElemValue() + switch yys3 { + case "PodeRestartThresholds": + if r.TryDecodeAsNil() { + x.PodeRestartThresholds = 0 + } else { + yyv4 := &x.PodeRestartThresholds + yym5 := z.DecBinary() + _ = yym5 + if false { + } else { + *((*int32)(yyv4)) = int32(r.DecodeInt(32)) + } + } + case "IncludingInitContainers": + if r.TryDecodeAsNil() { + x.IncludingInitContainers = false + } else { + yyv6 := &x.IncludingInitContainers + yym7 := z.DecBinary() + _ = yym7 + if false { + } else { + *((*bool)(yyv6)) = r.DecodeBool() + } + } + default: + z.DecStructFieldNotFound(-1, yys3) + } // end switch yys3 + } // end for yyj3 + r.ReadMapEnd() +} + +func (x *PodsHavingTooManyRestarts) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyj8 int + var yyb8 bool + var yyhl8 bool = l >= 0 + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + r.ReadArrayEnd() + return + } + r.ReadArrayElem() + if r.TryDecodeAsNil() { + x.PodeRestartThresholds = 0 + } else { + yyv9 := &x.PodeRestartThresholds + yym10 := z.DecBinary() + _ = yym10 + if false { + } else { + *((*int32)(yyv9)) = int32(r.DecodeInt(32)) + } + } + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + r.ReadArrayEnd() + return + } + r.ReadArrayElem() + if r.TryDecodeAsNil() { + x.IncludingInitContainers = false + } else { + yyv11 := &x.IncludingInitContainers + yym12 := z.DecBinary() + _ = yym12 + if false { + } else { + *((*bool)(yyv11)) = r.DecodeBool() + } + } + for { + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + break + } + r.ReadArrayElem() + z.DecStructFieldNotFound(yyj8-1, "") + } + r.ReadArrayEnd() +} + func (x codecSelfer1234) encStrategyList(v StrategyList, e *codec1978.Encoder) { var h codecSelfer1234 z, r := codec1978.GenHelperEncoder(e) @@ -1192,7 +1444,7 @@ func (x codecSelfer1234) decStrategyList(v *StrategyList, d *codec1978.Decoder) yyl1 := r.ReadMapStart() yybh1 := z.DecBasicHandle() if yyv1 == nil { - yyrl1 := z.DecInferLen(yyl1, yybh1.MaxInitLen, 80) + yyrl1 := z.DecInferLen(yyl1, yybh1.MaxInitLen, 88) yyv1 = make(map[StrategyName]DeschedulerStrategy, yyrl1) *v = yyv1 } diff --git a/pkg/api/types.go b/pkg/api/types.go index 5b1d1c30ce..a37fafa027 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -48,6 +48,7 @@ type DeschedulerStrategy struct { type StrategyParameters struct { NodeResourceUtilizationThresholds NodeResourceUtilizationThresholds NodeAffinityType []string + PodsHavingTooManyRestarts PodsHavingTooManyRestarts } type Percentage float64 @@ -58,3 +59,8 @@ type NodeResourceUtilizationThresholds struct { TargetThresholds ResourceThresholds NumberOfNodes int } + +type PodsHavingTooManyRestarts struct { + PodeRestartThresholds int32 + IncludingInitContainers bool +} diff --git a/pkg/api/v1alpha1/types.generated.go b/pkg/api/v1alpha1/types.generated.go index 25c569a0d0..13e654da5e 100644 --- a/pkg/api/v1alpha1/types.generated.go +++ b/pkg/api/v1alpha1/types.generated.go @@ -707,14 +707,15 @@ func (x *StrategyParameters) CodecEncodeSelf(e *codec1978.Encoder) { } else { yysep2 := !z.EncBinary() yy2arr2 := z.EncBasicHandle().StructToArray - var yyq2 [2]bool + var yyq2 [3]bool _ = yyq2 _, _ = yysep2, yy2arr2 const yyr2 bool = false yyq2[0] = true yyq2[1] = len(x.NodeAffinityType) != 0 + yyq2[2] = true if yyr2 || yy2arr2 { - r.WriteArrayStart(2) + r.WriteArrayStart(3) } else { var yynn2 = 0 for _, b := range yyq2 { @@ -775,6 +776,23 @@ func (x *StrategyParameters) CodecEncodeSelf(e *codec1978.Encoder) { } } } + if yyr2 || yy2arr2 { + r.WriteArrayElem() + if yyq2[2] { + yy12 := &x.PodsHavingTooManyRestarts + yy12.CodecEncodeSelf(e) + } else { + r.EncodeNil() + } + } else { + if yyq2[2] { + r.WriteMapElemKey() + r.EncStructFieldKey(codecSelferValueTypeString1234, `podsHavingTooManyRestarts`) + r.WriteMapElemValue() + yy14 := &x.PodsHavingTooManyRestarts + yy14.CodecEncodeSelf(e) + } + } if yyr2 || yy2arr2 { r.WriteArrayEnd() } else { @@ -853,6 +871,13 @@ func (x *StrategyParameters) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) z.F.DecSliceStringX(yyv5, d) } } + case "podsHavingTooManyRestarts": + if r.TryDecodeAsNil() { + x.PodsHavingTooManyRestarts = PodsHavingTooManyRestarts{} + } else { + yyv7 := &x.PodsHavingTooManyRestarts + yyv7.CodecDecodeSelf(d) + } default: z.DecStructFieldNotFound(-1, yys3) } // end switch yys3 @@ -864,16 +889,16 @@ func (x *StrategyParameters) codecDecodeSelfFromArray(l int, d *codec1978.Decode var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj7 int - var yyb7 bool - var yyhl7 bool = l >= 0 - yyj7++ - if yyhl7 { - yyb7 = yyj7 > l + var yyj8 int + var yyb8 bool + var yyhl8 bool = l >= 0 + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l } else { - yyb7 = r.CheckBreak() + yyb8 = r.CheckBreak() } - if yyb7 { + if yyb8 { r.ReadArrayEnd() return } @@ -881,16 +906,16 @@ func (x *StrategyParameters) codecDecodeSelfFromArray(l int, d *codec1978.Decode if r.TryDecodeAsNil() { x.NodeResourceUtilizationThresholds = NodeResourceUtilizationThresholds{} } else { - yyv8 := &x.NodeResourceUtilizationThresholds - yyv8.CodecDecodeSelf(d) + yyv9 := &x.NodeResourceUtilizationThresholds + yyv9.CodecDecodeSelf(d) } - yyj7++ - if yyhl7 { - yyb7 = yyj7 > l + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l } else { - yyb7 = r.CheckBreak() + yyb8 = r.CheckBreak() } - if yyb7 { + if yyb8 { r.ReadArrayEnd() return } @@ -898,26 +923,43 @@ func (x *StrategyParameters) codecDecodeSelfFromArray(l int, d *codec1978.Decode if r.TryDecodeAsNil() { x.NodeAffinityType = nil } else { - yyv9 := &x.NodeAffinityType - yym10 := z.DecBinary() - _ = yym10 + yyv10 := &x.NodeAffinityType + yym11 := z.DecBinary() + _ = yym11 if false { } else { - z.F.DecSliceStringX(yyv9, d) + z.F.DecSliceStringX(yyv10, d) } } + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + r.ReadArrayEnd() + return + } + r.ReadArrayElem() + if r.TryDecodeAsNil() { + x.PodsHavingTooManyRestarts = PodsHavingTooManyRestarts{} + } else { + yyv12 := &x.PodsHavingTooManyRestarts + yyv12.CodecDecodeSelf(d) + } for { - yyj7++ - if yyhl7 { - yyb7 = yyj7 > l + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l } else { - yyb7 = r.CheckBreak() + yyb8 = r.CheckBreak() } - if yyb7 { + if yyb8 { break } r.ReadArrayElem() - z.DecStructFieldNotFound(yyj7-1, "") + z.DecStructFieldNotFound(yyj8-1, "") } r.ReadArrayEnd() } @@ -1258,6 +1300,246 @@ func (x *NodeResourceUtilizationThresholds) codecDecodeSelfFromArray(l int, d *c r.ReadArrayEnd() } +func (x *PodsHavingTooManyRestarts) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + if x == nil { + r.EncodeNil() + } else { + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if yyxt1 := z.Extension(z.I2Rtid(x)); yyxt1 != nil { + z.EncExtension(x, yyxt1) + } else { + yysep2 := !z.EncBinary() + yy2arr2 := z.EncBasicHandle().StructToArray + var yyq2 [2]bool + _ = yyq2 + _, _ = yysep2, yy2arr2 + const yyr2 bool = false + yyq2[0] = x.PodeRestartThresholds != 0 + yyq2[1] = x.IncludingInitContainers != false + if yyr2 || yy2arr2 { + r.WriteArrayStart(2) + } else { + var yynn2 = 0 + for _, b := range yyq2 { + if b { + yynn2++ + } + } + r.WriteMapStart(yynn2) + yynn2 = 0 + } + if yyr2 || yy2arr2 { + r.WriteArrayElem() + if yyq2[0] { + yym4 := z.EncBinary() + _ = yym4 + if false { + } else { + r.EncodeInt(int64(x.PodeRestartThresholds)) + } + } else { + r.EncodeInt(0) + } + } else { + if yyq2[0] { + r.WriteMapElemKey() + r.EncStructFieldKey(codecSelferValueTypeString1234, `podeRestartThresholds`) + r.WriteMapElemValue() + yym5 := z.EncBinary() + _ = yym5 + if false { + } else { + r.EncodeInt(int64(x.PodeRestartThresholds)) + } + } + } + if yyr2 || yy2arr2 { + r.WriteArrayElem() + if yyq2[1] { + yym7 := z.EncBinary() + _ = yym7 + if false { + } else { + r.EncodeBool(bool(x.IncludingInitContainers)) + } + } else { + r.EncodeBool(false) + } + } else { + if yyq2[1] { + r.WriteMapElemKey() + r.EncStructFieldKey(codecSelferValueTypeString1234, `includingInitContainers`) + r.WriteMapElemValue() + yym8 := z.EncBinary() + _ = yym8 + if false { + } else { + r.EncodeBool(bool(x.IncludingInitContainers)) + } + } + } + if yyr2 || yy2arr2 { + r.WriteArrayEnd() + } else { + r.WriteMapEnd() + } + } + } +} + +func (x *PodsHavingTooManyRestarts) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if yyxt1 := z.Extension(z.I2Rtid(x)); yyxt1 != nil { + z.DecExtension(x, yyxt1) + } else { + yyct2 := r.ContainerType() + if yyct2 == codecSelferValueTypeMap1234 { + yyl2 := r.ReadMapStart() + if yyl2 == 0 { + r.ReadMapEnd() + } else { + x.codecDecodeSelfFromMap(yyl2, d) + } + } else if yyct2 == codecSelferValueTypeArray1234 { + yyl2 := r.ReadArrayStart() + if yyl2 == 0 { + r.ReadArrayEnd() + } else { + x.codecDecodeSelfFromArray(yyl2, d) + } + } else { + panic(errCodecSelferOnlyMapOrArrayEncodeToStruct1234) + } + } +} + +func (x *PodsHavingTooManyRestarts) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyhl3 bool = l >= 0 + for yyj3 := 0; ; yyj3++ { + if yyhl3 { + if yyj3 >= l { + break + } + } else { + if r.CheckBreak() { + break + } + } + r.ReadMapElemKey() + yys3 := z.StringView(r.DecStructFieldKey(codecSelferValueTypeString1234, z.DecScratchArrayBuffer())) + r.ReadMapElemValue() + switch yys3 { + case "podeRestartThresholds": + if r.TryDecodeAsNil() { + x.PodeRestartThresholds = 0 + } else { + yyv4 := &x.PodeRestartThresholds + yym5 := z.DecBinary() + _ = yym5 + if false { + } else { + *((*int32)(yyv4)) = int32(r.DecodeInt(32)) + } + } + case "includingInitContainers": + if r.TryDecodeAsNil() { + x.IncludingInitContainers = false + } else { + yyv6 := &x.IncludingInitContainers + yym7 := z.DecBinary() + _ = yym7 + if false { + } else { + *((*bool)(yyv6)) = r.DecodeBool() + } + } + default: + z.DecStructFieldNotFound(-1, yys3) + } // end switch yys3 + } // end for yyj3 + r.ReadMapEnd() +} + +func (x *PodsHavingTooManyRestarts) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyj8 int + var yyb8 bool + var yyhl8 bool = l >= 0 + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + r.ReadArrayEnd() + return + } + r.ReadArrayElem() + if r.TryDecodeAsNil() { + x.PodeRestartThresholds = 0 + } else { + yyv9 := &x.PodeRestartThresholds + yym10 := z.DecBinary() + _ = yym10 + if false { + } else { + *((*int32)(yyv9)) = int32(r.DecodeInt(32)) + } + } + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + r.ReadArrayEnd() + return + } + r.ReadArrayElem() + if r.TryDecodeAsNil() { + x.IncludingInitContainers = false + } else { + yyv11 := &x.IncludingInitContainers + yym12 := z.DecBinary() + _ = yym12 + if false { + } else { + *((*bool)(yyv11)) = r.DecodeBool() + } + } + for { + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + break + } + r.ReadArrayElem() + z.DecStructFieldNotFound(yyj8-1, "") + } + r.ReadArrayEnd() +} + func (x codecSelfer1234) encStrategyList(v StrategyList, e *codec1978.Encoder) { var h codecSelfer1234 z, r := codec1978.GenHelperEncoder(e) @@ -1282,7 +1564,7 @@ func (x codecSelfer1234) decStrategyList(v *StrategyList, d *codec1978.Decoder) yyl1 := r.ReadMapStart() yybh1 := z.DecBasicHandle() if yyv1 == nil { - yyrl1 := z.DecInferLen(yyl1, yybh1.MaxInitLen, 80) + yyrl1 := z.DecInferLen(yyl1, yybh1.MaxInitLen, 88) yyv1 = make(map[StrategyName]DeschedulerStrategy, yyrl1) *v = yyv1 } diff --git a/pkg/api/v1alpha1/types.go b/pkg/api/v1alpha1/types.go index 1ca0cac9bd..c45e315eac 100644 --- a/pkg/api/v1alpha1/types.go +++ b/pkg/api/v1alpha1/types.go @@ -48,6 +48,7 @@ type DeschedulerStrategy struct { type StrategyParameters struct { NodeResourceUtilizationThresholds NodeResourceUtilizationThresholds `json:"nodeResourceUtilizationThresholds,omitempty"` NodeAffinityType []string `json:"nodeAffinityType,omitempty"` + PodsHavingTooManyRestarts PodsHavingTooManyRestarts `json:"podsHavingTooManyRestarts,omitempty"` } type Percentage float64 @@ -58,3 +59,8 @@ type NodeResourceUtilizationThresholds struct { TargetThresholds ResourceThresholds `json:"targetThresholds,omitempty"` NumberOfNodes int `json:"numberOfNodes,omitempty"` } + +type PodsHavingTooManyRestarts struct { + PodeRestartThresholds int32 `json:"podeRestartThresholds,omitempty"` + IncludingInitContainers bool `json:"includingInitContainers,omitempty"` +} diff --git a/pkg/api/v1alpha1/zz_generated.conversion.go b/pkg/api/v1alpha1/zz_generated.conversion.go index 315bc6860b..a6e2a47e7b 100644 --- a/pkg/api/v1alpha1/zz_generated.conversion.go +++ b/pkg/api/v1alpha1/zz_generated.conversion.go @@ -42,6 +42,8 @@ func RegisterConversions(scheme *runtime.Scheme) error { Convert_api_DeschedulerStrategy_To_v1alpha1_DeschedulerStrategy, Convert_v1alpha1_NodeResourceUtilizationThresholds_To_api_NodeResourceUtilizationThresholds, Convert_api_NodeResourceUtilizationThresholds_To_v1alpha1_NodeResourceUtilizationThresholds, + Convert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts, + Convert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts, Convert_v1alpha1_StrategyParameters_To_api_StrategyParameters, Convert_api_StrategyParameters_To_v1alpha1_StrategyParameters, ) @@ -119,11 +121,36 @@ func Convert_api_NodeResourceUtilizationThresholds_To_v1alpha1_NodeResourceUtili return autoConvert_api_NodeResourceUtilizationThresholds_To_v1alpha1_NodeResourceUtilizationThresholds(in, out, s) } +func autoConvert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts(in *PodsHavingTooManyRestarts, out *api.PodsHavingTooManyRestarts, s conversion.Scope) error { + out.PodeRestartThresholds = in.PodeRestartThresholds + out.IncludingInitContainers = in.IncludingInitContainers + return nil +} + +// Convert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts is an autogenerated conversion function. +func Convert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts(in *PodsHavingTooManyRestarts, out *api.PodsHavingTooManyRestarts, s conversion.Scope) error { + return autoConvert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts(in, out, s) +} + +func autoConvert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts(in *api.PodsHavingTooManyRestarts, out *PodsHavingTooManyRestarts, s conversion.Scope) error { + out.PodeRestartThresholds = in.PodeRestartThresholds + out.IncludingInitContainers = in.IncludingInitContainers + return nil +} + +// Convert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts is an autogenerated conversion function. +func Convert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts(in *api.PodsHavingTooManyRestarts, out *PodsHavingTooManyRestarts, s conversion.Scope) error { + return autoConvert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts(in, out, s) +} + func autoConvert_v1alpha1_StrategyParameters_To_api_StrategyParameters(in *StrategyParameters, out *api.StrategyParameters, s conversion.Scope) error { if err := Convert_v1alpha1_NodeResourceUtilizationThresholds_To_api_NodeResourceUtilizationThresholds(&in.NodeResourceUtilizationThresholds, &out.NodeResourceUtilizationThresholds, s); err != nil { return err } out.NodeAffinityType = *(*[]string)(unsafe.Pointer(&in.NodeAffinityType)) + if err := Convert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts(&in.PodsHavingTooManyRestarts, &out.PodsHavingTooManyRestarts, s); err != nil { + return err + } return nil } @@ -137,6 +164,9 @@ func autoConvert_api_StrategyParameters_To_v1alpha1_StrategyParameters(in *api.S return err } out.NodeAffinityType = *(*[]string)(unsafe.Pointer(&in.NodeAffinityType)) + if err := Convert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts(&in.PodsHavingTooManyRestarts, &out.PodsHavingTooManyRestarts, s); err != nil { + return err + } return nil } diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index eccebf83fe..fc5c9f1b53 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -106,6 +106,22 @@ func (in *NodeResourceUtilizationThresholds) DeepCopy() *NodeResourceUtilization return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodsHavingTooManyRestarts) DeepCopyInto(out *PodsHavingTooManyRestarts) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodsHavingTooManyRestarts. +func (in *PodsHavingTooManyRestarts) DeepCopy() *PodsHavingTooManyRestarts { + if in == nil { + return nil + } + out := new(PodsHavingTooManyRestarts) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StrategyParameters) DeepCopyInto(out *StrategyParameters) { *out = *in @@ -115,6 +131,7 @@ func (in *StrategyParameters) DeepCopyInto(out *StrategyParameters) { *out = make([]string, len(*in)) copy(*out, *in) } + out.PodsHavingTooManyRestarts = in.PodsHavingTooManyRestarts return } diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index ccbbe6cb64..99a73e6b86 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -106,6 +106,22 @@ func (in *NodeResourceUtilizationThresholds) DeepCopy() *NodeResourceUtilization return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodsHavingTooManyRestarts) DeepCopyInto(out *PodsHavingTooManyRestarts) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodsHavingTooManyRestarts. +func (in *PodsHavingTooManyRestarts) DeepCopy() *PodsHavingTooManyRestarts { + if in == nil { + return nil + } + out := new(PodsHavingTooManyRestarts) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StrategyParameters) DeepCopyInto(out *StrategyParameters) { *out = *in @@ -115,6 +131,7 @@ func (in *StrategyParameters) DeepCopyInto(out *StrategyParameters) { *out = make([]string, len(*in)) copy(*out, *in) } + out.PodsHavingTooManyRestarts = in.PodsHavingTooManyRestarts return } diff --git a/pkg/descheduler/descheduler.go b/pkg/descheduler/descheduler.go index 07d73c395b..5f18d6c785 100644 --- a/pkg/descheduler/descheduler.go +++ b/pkg/descheduler/descheduler.go @@ -65,6 +65,7 @@ func Run(rs *options.DeschedulerServer) error { strategies.LowNodeUtilization(rs, deschedulerPolicy.Strategies["LowNodeUtilization"], evictionPolicyGroupVersion, nodes, nodePodCount) strategies.RemovePodsViolatingInterPodAntiAffinity(rs, deschedulerPolicy.Strategies["RemovePodsViolatingInterPodAntiAffinity"], evictionPolicyGroupVersion, nodes, nodePodCount) strategies.RemovePodsViolatingNodeAffinity(rs, deschedulerPolicy.Strategies["RemovePodsViolatingNodeAffinity"], evictionPolicyGroupVersion, nodes, nodePodCount) + strategies.RemovePodsHavingTooManyRestarts(rs, deschedulerPolicy.Strategies["RemovePodsHavingTooManyRestarts"], evictionPolicyGroupVersion, nodes, nodePodCount) return nil } diff --git a/pkg/descheduler/strategies/toomanyrestarts.go b/pkg/descheduler/strategies/toomanyrestarts.go new file mode 100644 index 0000000000..766516c641 --- /dev/null +++ b/pkg/descheduler/strategies/toomanyrestarts.go @@ -0,0 +1,98 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strategies + +import ( + "github.com/golang/glog" + + "k8s.io/api/core/v1" + + "github.com/kubernetes-incubator/descheduler/cmd/descheduler/app/options" + "github.com/kubernetes-incubator/descheduler/pkg/api" + "github.com/kubernetes-incubator/descheduler/pkg/descheduler/evictions" + podutil "github.com/kubernetes-incubator/descheduler/pkg/descheduler/pod" +) + +// RemovePodsHavingTooManyRestarts removes the pods that have too many restarts on node. +// There are too many cases leading this issue: Volume mount failed, app error due to nodes' different settings. +// As of now, this strategy won't evict daemonsets, mirror pods, critical pods and pods with local storages. +func RemovePodsHavingTooManyRestarts(ds *options.DeschedulerServer, strategy api.DeschedulerStrategy, policyGroupVersion string, nodes []*v1.Node, nodePodCount nodePodEvictedCount) { + evictionCount := removePodsHavingTooManyRestarts(ds, strategy, policyGroupVersion, nodes, nodePodCount) + glog.V(1).Infof("RemovePodsHavingTooManyRestarts evicted %v pods", evictionCount) +} + +func removePodsHavingTooManyRestarts(ds *options.DeschedulerServer, strategy api.DeschedulerStrategy, policyGroupVersion string, nodes []*v1.Node, nodePodCount nodePodEvictedCount) int { + podsEvicted := 0 + + if !strategy.Enabled || strategy.Params.PodsHavingTooManyRestarts.PodeRestartThresholds < 1 { + glog.Info("RemovePodsHavingTooManyRestarts policy is disabled.") + return podsEvicted + } + + for _, node := range nodes { + glog.V(1).Infof("RemovePodsHavingTooManyRestarts Processing node: %#v", node.Name) + pods, err := podutil.ListEvictablePodsOnNode(ds.Client, node) + + if err != nil { + glog.Errorf("RemovePodsHavingTooManyRestarts Error when list pods at node %s", node.Name) + continue + } + for _, pod := range pods { + if ds.MaxNoOfPodsToEvictPerNode > 0 && nodePodCount[node]+1 > ds.MaxNoOfPodsToEvictPerNode { + break + } + + restarts, initRestarts := calcContainerRestarts(pod) + if strategy.Params.PodsHavingTooManyRestarts.IncludingInitContainers { + if restarts+initRestarts <= strategy.Params.PodsHavingTooManyRestarts.PodeRestartThresholds { + continue + } + } else if restarts <= strategy.Params.PodsHavingTooManyRestarts.PodeRestartThresholds { + continue + } + + glog.V(1).Infof("RemovePodsHavingTooManyRestarts will evicted pod: %#v, container restarts: %d, initContainer restarts: %d", pod.Name, restarts, initRestarts) + success, err := evictions.EvictPod(ds.Client, pod, policyGroupVersion, ds.DryRun) + if !success { + glog.Infof("RemovePodsHavingTooManyRestarts Error when evicting pod: %#v (%#v)", pod.Name, err) + } else { + nodePodCount[node]++ + glog.V(1).Infof("RemovePodsHavingTooManyRestarts Evicted pod: %#v (%#v)", pod.Name, err) + } + } + podsEvicted += nodePodCount[node] + } + return podsEvicted +} + +// calcContainerRestarts get container restarts and init container restarts. +func calcContainerRestarts(pod *v1.Pod) (int32, int32) { + var ( + restarts int32 = 0 + initRestarts int32 = 0 + ) + + for _, cs := range pod.Status.ContainerStatuses { + restarts += cs.RestartCount + } + + for _, cs := range pod.Status.InitContainerStatuses { + initRestarts += cs.RestartCount + } + + return restarts, initRestarts +} diff --git a/pkg/descheduler/strategies/toomanyrestarts_test.go b/pkg/descheduler/strategies/toomanyrestarts_test.go new file mode 100644 index 0000000000..0adb270bfd --- /dev/null +++ b/pkg/descheduler/strategies/toomanyrestarts_test.go @@ -0,0 +1,209 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strategies + +import ( + "testing" + + "fmt" + "github.com/kubernetes-incubator/descheduler/cmd/descheduler/app/options" + "github.com/kubernetes-incubator/descheduler/pkg/api" + "github.com/kubernetes-incubator/descheduler/pkg/apis/componentconfig" + "github.com/kubernetes-incubator/descheduler/test" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" +) + +func initPods(node *v1.Node) []v1.Pod { + + pods := make([]v1.Pod, 0) + + var i int32 = 0 + for i <= 10 { + pod := test.BuildTestPod(fmt.Sprintf("pod-%d", i), 100, 0, node.Name) + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + + // pod i will has 25 * i restarts. + pod.Status = v1.PodStatus{ + InitContainerStatuses: []v1.ContainerStatus{ + { + RestartCount: 5 * i, + }, + }, + ContainerStatuses: []v1.ContainerStatus{ + { + RestartCount: 10 * i, + }, + { + RestartCount: 10 * i, + }, + }, + } + pods = append(pods, *pod) + i++ + } + + // The following 4 pods won't get evicted. + // A daemonset. + pods[6].ObjectMeta.OwnerReferences = test.GetDaemonSetOwnerRefList() + // A pod with local storage. + pods[7].ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pods[7].Spec.Volumes = []v1.Volume{ + { + Name: "sample", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)}, + }, + }, + } + // A Mirror Pod. + pods[8].Annotations = test.GetMirrorPodAnnotation() + // A Critical Pod. + pods[9].Namespace = "kube-system" + pods[9].Annotations = test.GetCriticalPodAnnotation() + + return pods +} + +func TestRemovePodsHavingTooManyRestarts(t *testing.T) { + + createStrategy := func(enabled, includingInitContainers bool, restartThresholds int32) api.DeschedulerStrategy { + return api.DeschedulerStrategy{ + Enabled: enabled, + Params: api.StrategyParameters{ + PodsHavingTooManyRestarts: api.PodsHavingTooManyRestarts{ + PodeRestartThresholds: restartThresholds, + IncludingInitContainers: includingInitContainers, + }, + }, + } + } + + node := test.BuildTestNode("node1", 2000, 3000, 10) + pods := initPods(node) + + fakeClient := &fake.Clientset{} + + fakeClient.Fake.AddReactor("list", "pods", func(action core.Action) (bool, runtime.Object, error) { + return true, &v1.PodList{Items: pods}, nil + }) + + tests := []struct { + description string + pods []v1.Pod + strategy api.DeschedulerStrategy + expectedEvictedPodCount int + maxPodsToEvict int + }{ + { + description: "Not a PodsHavingTooManyRestarts strategy, no pod evictions", + strategy: api.DeschedulerStrategy{ + Enabled: true, + Params: api.StrategyParameters{ + NodeAffinityType: []string{ + "requiredDuringSchedulingRequiredDuringExecution", + }, + }, + }, + expectedEvictedPodCount: 0, + maxPodsToEvict: 0, + }, + { + description: "Strategy is disabled, no pod evictions", + strategy: createStrategy(false, true, 10), + expectedEvictedPodCount: 0, + maxPodsToEvict: 0, + }, + { + description: "All pods have total restarts under threshold, no pod evictions", + strategy: createStrategy(true, true, 10000), + expectedEvictedPodCount: 0, + maxPodsToEvict: 0, + }, + { + description: "One pod have total restarts bigger than threshold, 6 pod evictions", + strategy: createStrategy(true, true, 1), + expectedEvictedPodCount: 6, + maxPodsToEvict: 0, + }, + { + description: "Nine pods have total restarts equals threshold(includingInitContainers=true), 5 pods evictions", + strategy: createStrategy(true, true, 1*25), + expectedEvictedPodCount: 5, + maxPodsToEvict: 0, + }, + { + description: "Nine pods have total restarts equals threshold(includingInitContainers=false), 5 pods evictions", + strategy: createStrategy(true, false, 1*25), + expectedEvictedPodCount: 5, + maxPodsToEvict: 0, + }, + { + description: "All pods have total restarts equals threshold(includingInitContainers=true), 6 pods evictions", + strategy: createStrategy(true, true, 1*20), + expectedEvictedPodCount: 6, + maxPodsToEvict: 0, + }, + { + description: "Nine pods have total restarts equals threshold(includingInitContainers=false), 5 pods evictions", + strategy: createStrategy(true, false, 1*20), + expectedEvictedPodCount: 5, + maxPodsToEvict: 0, + }, + { + description: "Five pods have total restarts bigger than threshold(includingInitContainers=true), but only 1 pod eviction", + strategy: createStrategy(true, true, 5*25+1), + expectedEvictedPodCount: 1, + maxPodsToEvict: 0, + }, + { + description: "Five pods have total restarts bigger than threshold(includingInitContainers=false), but only 1 pod eviction", + strategy: createStrategy(true, false, 5*20+1), + expectedEvictedPodCount: 1, + maxPodsToEvict: 0, + }, + { + description: "All pods have total restarts equals threshold(maxPodsToEvict=3), 3 pods evictions", + strategy: createStrategy(true, true, 1), + expectedEvictedPodCount: 3, + maxPodsToEvict: 3, + }, + } + + for _, tc := range tests { + + ds := options.DeschedulerServer{ + DeschedulerConfiguration: componentconfig.DeschedulerConfiguration{ + MaxNoOfPodsToEvictPerNode: tc.maxPodsToEvict, + }, + Client: fakeClient, + } + + nodePodCount := InitializeNodePodCount([]*v1.Node{node}) + + actualEvictedPodCount := removePodsHavingTooManyRestarts(&ds, tc.strategy, "v1", []*v1.Node{node}, nodePodCount) + if actualEvictedPodCount != tc.expectedEvictedPodCount { + t.Errorf("Test %#v failed, expected %v pod evictions, but got %v pod evictions\n", tc.description, tc.expectedEvictedPodCount, actualEvictedPodCount) + } + } + +}