From 737c0ec9eb99d895bf141f3701aa8e1e4100a445 Mon Sep 17 00:00:00 2001 From: xiangpengzhao Date: Thu, 23 Nov 2017 16:58:49 +0800 Subject: [PATCH 1/5] Add validation of kubelet configuration in kubeadm. --- .../app/apis/kubeadm/validation/validation.go | 32 +++++++++ .../kubeadm/validation/validation_test.go | 71 +++++++++++++++++-- 2 files changed, 99 insertions(+), 4 deletions(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index 3fad8dff3b796..df72c28ec73c1 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -36,6 +36,9 @@ import ( tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token" apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" + "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" + kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" + kubeletvalidation "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/validation" "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig" kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme" proxyvalidation "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/validation" @@ -75,6 +78,9 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList allErrs = append(allErrs, ValidateFeatureGates(c.FeatureGates, field.NewPath("feature-gates"))...) allErrs = append(allErrs, ValidateAPIEndpoint(c, field.NewPath("api-endpoint"))...) //allErrs = append(allErrs, ValidateProxy(c, field.NewPath("kube-proxy"))...) + if features.Enabled(c.FeatureGates, features.DynamicKubeletConfig) { + allErrs = append(allErrs, ValidateKubeletConfiguration(&c.KubeletConfiguration, field.NewPath("kubeletConfiguration"))...) + } return allErrs } @@ -369,3 +375,29 @@ func ValidateIgnorePreflightErrors(ignorePreflightErrors []string, skipPreflight return ignoreErrors, allErrs.ToAggregate() } + +// ValidateKubeletConfiguration validates kubelet configuration and collects all encountered errors +func ValidateKubeletConfiguration(c *kubeadm.KubeletConfiguration, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + scheme, _, err := kubeletscheme.NewSchemeAndCodecs() + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, "kubeletConfiguration", err.Error())) + return allErrs + } + + // Convert versioned config to internal config + internalcfg := &kubeletconfig.KubeletConfiguration{} + err = scheme.Convert(c.BaseConfig, internalcfg, nil) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, "kubeletConfiguration", err.Error())) + return allErrs + } + + err = kubeletvalidation.ValidateKubeletConfiguration(internalcfg) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, "kubeletConfiguration", err.Error())) + } + + return allErrs +} diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go index 37678de375798..826debfe42606 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go @@ -25,8 +25,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1" - "k8s.io/kubernetes/pkg/util/pointer" + utilpointer "k8s.io/kubernetes/pkg/util/pointer" ) func TestValidateTokenDiscovery(t *testing.T) { @@ -338,9 +339,9 @@ func TestValidateMasterConfiguration(t *testing.T) { KubeProxy: kubeadm.KubeProxy{ Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{ Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{ - Max: pointer.Int32Ptr(2), - MaxPerCore: pointer.Int32Ptr(1), - Min: pointer.Int32Ptr(1), + Max: utilpointer.Int32Ptr(2), + MaxPerCore: utilpointer.Int32Ptr(1), + Min: utilpointer.Int32Ptr(1), TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second}, TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second}, }, @@ -502,3 +503,65 @@ func TestValidateIgnorePreflightErrors(t *testing.T) { } } } + +func TestValidateKubeletConfiguration(t *testing.T) { + successCase := &kubeadm.KubeletConfiguration{ + BaseConfig: &kubeletconfigv1alpha1.KubeletConfiguration{ + CgroupsPerQOS: utilpointer.BoolPtr(true), + EnforceNodeAllocatable: []string{"pods", "system-reserved", "kube-reserved"}, + SystemCgroups: "", + CgroupRoot: "", + CAdvisorPort: utilpointer.Int32Ptr(0), + EventBurst: 10, + EventRecordQPS: utilpointer.Int32Ptr(5), + HealthzPort: utilpointer.Int32Ptr(10248), + ImageGCHighThresholdPercent: utilpointer.Int32Ptr(85), + ImageGCLowThresholdPercent: utilpointer.Int32Ptr(80), + IPTablesDropBit: utilpointer.Int32Ptr(15), + IPTablesMasqueradeBit: utilpointer.Int32Ptr(14), + KubeAPIBurst: 10, + KubeAPIQPS: utilpointer.Int32Ptr(5), + MaxOpenFiles: 1000000, + MaxPods: 110, + OOMScoreAdj: utilpointer.Int32Ptr(-999), + PodsPerCore: 100, + Port: 65535, + ReadOnlyPort: utilpointer.Int32Ptr(0), + RegistryBurst: 10, + RegistryPullQPS: utilpointer.Int32Ptr(5), + }, + } + if allErrors := ValidateKubeletConfiguration(successCase, nil); len(allErrors) != 0 { + t.Errorf("failed ValidateKubeletConfiguration: expect no errors but got %v", allErrors) + } + + errorCase := &kubeadm.KubeletConfiguration{ + BaseConfig: &kubeletconfigv1alpha1.KubeletConfiguration{ + CgroupsPerQOS: utilpointer.BoolPtr(false), + EnforceNodeAllocatable: []string{"pods", "system-reserved", "kube-reserved", "illegal-key"}, + SystemCgroups: "/", + CgroupRoot: "", + CAdvisorPort: utilpointer.Int32Ptr(-10), + EventBurst: -10, + EventRecordQPS: utilpointer.Int32Ptr(-10), + HealthzPort: utilpointer.Int32Ptr(-10), + ImageGCHighThresholdPercent: utilpointer.Int32Ptr(101), + ImageGCLowThresholdPercent: utilpointer.Int32Ptr(101), + IPTablesDropBit: utilpointer.Int32Ptr(-10), + IPTablesMasqueradeBit: utilpointer.Int32Ptr(-10), + KubeAPIBurst: -10, + KubeAPIQPS: utilpointer.Int32Ptr(-10), + MaxOpenFiles: -10, + MaxPods: -10, + OOMScoreAdj: utilpointer.Int32Ptr(-1001), + PodsPerCore: -10, + Port: 0, + ReadOnlyPort: utilpointer.Int32Ptr(-10), + RegistryBurst: -10, + RegistryPullQPS: utilpointer.Int32Ptr(-10), + }, + } + if allErrors := ValidateKubeletConfiguration(errorCase, nil); len(allErrors) == 0 { + t.Errorf("failed ValidateKubeletConfiguration: expect errors but got no error") + } +} From 845a10f0a9444e55ebcf8150328528e2641bd400 Mon Sep 17 00:00:00 2001 From: xiangpengzhao Date: Thu, 23 Nov 2017 16:59:59 +0800 Subject: [PATCH 2/5] Auto generated BUILD files. --- cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD | 11 +++-- cmd/kubeadm/app/apis/kubeadm/validation/BUILD | 44 +++++++++---------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD index 35eb0bd7f11e8..70c1dee7788eb 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD @@ -1,9 +1,4 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", @@ -23,9 +18,12 @@ go_library( "//conditions:default": [], }), importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1", + visibility = ["//visibility:public"], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library", @@ -48,4 +46,5 @@ filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], + visibility = ["//visibility:public"], ) diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/BUILD b/cmd/kubeadm/app/apis/kubeadm/validation/BUILD index 9bf044689589b..75cbf7351292f 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/BUILD +++ b/cmd/kubeadm/app/apis/kubeadm/validation/BUILD @@ -1,30 +1,10 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["validation_test.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation", - library = ":go_default_library", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library", - "//pkg/util/pointer:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", - ], -) +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = ["validation.go"], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation", + visibility = ["//visibility:public"], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", @@ -33,6 +13,9 @@ go_library( "//cmd/kubeadm/app/util/token:go_default_library", "//pkg/apis/core/validation:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library", + "//pkg/kubelet/apis/kubeletconfig:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/validation:go_default_library", "//pkg/proxy/apis/kubeproxyconfig:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/validation:go_default_library", @@ -45,6 +28,22 @@ go_library( ], ) +go_test( + name = "go_default_test", + srcs = ["validation_test.go"], + importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation", + library = ":go_default_library", + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", + "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library", + "//pkg/util/pointer:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + ], +) + filegroup( name = "package-srcs", srcs = glob(["**"]), @@ -56,4 +55,5 @@ filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], + visibility = ["//visibility:public"], ) From da971d802039d138de99cc6b1eace1aee9544b42 Mon Sep 17 00:00:00 2001 From: xiangpengzhao Date: Sat, 25 Nov 2017 00:25:37 +0800 Subject: [PATCH 3/5] Only set defaults when DynamicKubeletConfig feature gate is on. --- cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go index f621fa0931f31..86548a0f0e4a3 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go @@ -110,7 +110,9 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) { } SetDefaultsEtcdSelfHosted(obj) - SetDefaults_KubeletConfiguration(obj) + if features.Enabled(obj.FeatureGates, features.DynamicKubeletConfig) { + SetDefaults_KubeletConfiguration(obj) + } SetDefaults_ProxyConfiguration(obj) } From 1e26acfa52e0b3232cbd69facab4df50c6961480 Mon Sep 17 00:00:00 2001 From: xiangpengzhao Date: Sat, 25 Nov 2017 00:47:55 +0800 Subject: [PATCH 4/5] Enable kube-proxy validation in kubeadm validation. --- .../app/apis/kubeadm/validation/validation.go | 2 +- .../kubeadm/validation/validation_test.go | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index df72c28ec73c1..f7b9ab2107282 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -77,7 +77,7 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...) allErrs = append(allErrs, ValidateFeatureGates(c.FeatureGates, field.NewPath("feature-gates"))...) allErrs = append(allErrs, ValidateAPIEndpoint(c, field.NewPath("api-endpoint"))...) - //allErrs = append(allErrs, ValidateProxy(c, field.NewPath("kube-proxy"))...) + allErrs = append(allErrs, ValidateProxy(c, field.NewPath("kube-proxy"))...) if features.Enabled(c.FeatureGates, features.DynamicKubeletConfig) { allErrs = append(allErrs, ValidateKubeletConfiguration(&c.KubeletConfiguration, field.NewPath("kubeletConfiguration"))...) } diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go index 826debfe42606..3027970c509a6 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go @@ -338,6 +338,21 @@ func TestValidateMasterConfiguration(t *testing.T) { }, KubeProxy: kubeadm.KubeProxy{ Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{ + BindAddress: "192.168.59.103", + HealthzBindAddress: "0.0.0.0:10256", + MetricsBindAddress: "127.0.0.1:10249", + ClusterCIDR: "192.168.59.0/24", + UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, + ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, + IPTables: kubeproxyconfigv1alpha1.KubeProxyIPTablesConfiguration{ + MasqueradeAll: true, + SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, + }, + IPVS: kubeproxyconfigv1alpha1.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{ Max: utilpointer.Int32Ptr(2), MaxPerCore: utilpointer.Int32Ptr(1), @@ -362,6 +377,32 @@ func TestValidateMasterConfiguration(t *testing.T) { AdvertiseAddress: "1:2:3::4", BindPort: 3446, }, + KubeProxy: kubeadm.KubeProxy{ + Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{ + BindAddress: "192.168.59.103", + HealthzBindAddress: "0.0.0.0:10256", + MetricsBindAddress: "127.0.0.1:10249", + ClusterCIDR: "192.168.59.0/24", + UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, + ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, + IPTables: kubeproxyconfigv1alpha1.KubeProxyIPTablesConfiguration{ + MasqueradeAll: true, + SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, + }, + IPVS: kubeproxyconfigv1alpha1.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, + Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{ + Max: utilpointer.Int32Ptr(2), + MaxPerCore: utilpointer.Int32Ptr(1), + Min: utilpointer.Int32Ptr(1), + TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second}, + TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second}, + }, + }, + }, AuthorizationModes: []string{"Node", "RBAC"}, Networking: kubeadm.Networking{ ServiceSubnet: "2001:db8::1/98", From cf97d6f6ebad83ed52092e1b489a941d9620e041 Mon Sep 17 00:00:00 2001 From: xiangpengzhao Date: Sat, 25 Nov 2017 01:04:02 +0800 Subject: [PATCH 5/5] Run the kubeletconfig defaulter in kubeadm defaults --- cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go index 86548a0f0e4a3..88581a1dc0a9c 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go @@ -23,6 +23,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" + kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme" kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1" @@ -202,4 +204,9 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) { if obj.KubeletConfiguration.BaseConfig.CAdvisorPort == nil { obj.KubeletConfiguration.BaseConfig.CAdvisorPort = utilpointer.Int32Ptr(0) } + + scheme, _, _ := kubeletscheme.NewSchemeAndCodecs() + if scheme != nil { + scheme.Default(obj.KubeletConfiguration.BaseConfig) + } }