From b4aac59a8b4047b054533622a817c009722b4410 Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Mon, 12 Aug 2024 02:18:08 +0000 Subject: [PATCH 1/2] fix EOF during TLS handshake caused by health check Signed-off-by: zhangzujian --- .../kube-ovn/templates/controller-deploy.yaml | 10 +- charts/kube-ovn/templates/monitor-deploy.yaml | 18 ++-- charts/kube-ovn/templates/ovncni-ds.yaml | 18 ++-- cmd/cmdmain.go | 20 ++-- .../health_check.go} | 22 ++-- dist/images/Dockerfile | 2 +- dist/images/install.sh | 46 +++++--- pkg/server/server.go | 101 ------------------ 8 files changed, 86 insertions(+), 151 deletions(-) rename cmd/{controller_health_check/controller_health_check.go => health_check/health_check.go} (63%) delete mode 100644 pkg/server/server.go diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml index e48c8415d6e..04df6c23a25 100644 --- a/charts/kube-ovn/templates/controller-deploy.yaml +++ b/charts/kube-ovn/templates/controller-deploy.yaml @@ -187,19 +187,21 @@ spec: readinessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 - --tls={{- .Values.func.SECURE_SERVING }} periodSeconds: 3 - timeoutSeconds: 45 + timeoutSeconds: 1 livenessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 - --tls={{- .Values.func.SECURE_SERVING }} initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 - timeoutSeconds: 45 + timeoutSeconds: 1 resources: requests: cpu: {{ index .Values "kube-ovn-controller" "requests" "cpu" }} diff --git a/charts/kube-ovn/templates/monitor-deploy.yaml b/charts/kube-ovn/templates/monitor-deploy.yaml index 93336e6b684..7c91327fe12 100644 --- a/charts/kube-ovn/templates/monitor-deploy.yaml +++ b/charts/kube-ovn/templates/monitor-deploy.yaml @@ -125,17 +125,23 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10661 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 1 readinessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10661 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 1 nodeSelector: kubernetes.io/os: "linux" {{- with splitList "=" .Values.MASTER_NODES_LABEL }} diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml index 1a101e09b6f..5b4443b20d6 100644 --- a/charts/kube-ovn/templates/ovncni-ds.yaml +++ b/charts/kube-ovn/templates/ovncni-ds.yaml @@ -198,17 +198,23 @@ spec: failureThreshold: 3 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 1 livenessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 1 resources: requests: cpu: {{ index .Values "kube-ovn-cni" "requests" "cpu" }} diff --git a/cmd/cmdmain.go b/cmd/cmdmain.go index addbcce737e..a4166f2fb76 100644 --- a/cmd/cmdmain.go +++ b/cmd/cmdmain.go @@ -12,7 +12,7 @@ import ( "k8s.io/klog/v2" "github.com/kubeovn/kube-ovn/cmd/controller" - "github.com/kubeovn/kube-ovn/cmd/controller_health_check" + "github.com/kubeovn/kube-ovn/cmd/health_check" "github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller" "github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker" "github.com/kubeovn/kube-ovn/cmd/ovn_monitor" @@ -22,13 +22,13 @@ import ( ) const ( - CmdController = "kube-ovn-controller" - CmdMonitor = "kube-ovn-monitor" - CmdSpeaker = "kube-ovn-speaker" - CmdWebhook = "kube-ovn-webhook" - CmdControllerHealthCheck = "kube-ovn-controller-healthcheck" - CmdOvnLeaderChecker = "kube-ovn-leader-checker" - CmdOvnICController = "kube-ovn-ic-controller" + CmdController = "kube-ovn-controller" + CmdMonitor = "kube-ovn-monitor" + CmdSpeaker = "kube-ovn-speaker" + CmdWebhook = "kube-ovn-webhook" + CmdHealthCheck = "kube-ovn-healthcheck" + CmdOvnLeaderChecker = "kube-ovn-leader-checker" + CmdOvnICController = "kube-ovn-ic-controller" ) const timeFormat = "2006-01-02_15:04:05" @@ -102,8 +102,8 @@ func main() { speaker.CmdMain() case CmdWebhook: webhook.CmdMain() - case CmdControllerHealthCheck: - controller_health_check.CmdMain() + case CmdHealthCheck: + health_check.CmdMain() case CmdOvnLeaderChecker: ovn_leader_checker.CmdMain() case CmdOvnICController: diff --git a/cmd/controller_health_check/controller_health_check.go b/cmd/health_check/health_check.go similarity index 63% rename from cmd/controller_health_check/controller_health_check.go rename to cmd/health_check/health_check.go index 7e04769210f..45e59382768 100644 --- a/cmd/controller_health_check/controller_health_check.go +++ b/cmd/health_check/health_check.go @@ -1,7 +1,8 @@ -package controller_health_check +package health_check import ( "flag" + "net" "os" "time" @@ -12,7 +13,8 @@ import ( ) func CmdMain() { - tls := pflag.Bool("tls", false, "Whether kube-ovn-controller uses TLS") + port := pflag.Int32("port", 0, "Target port") + tls := pflag.Bool("tls", false, "Dial the server with TLS") klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) klog.InitFlags(klogFlags) @@ -32,18 +34,24 @@ func CmdMain() { pflag.CommandLine.AddGoFlagSet(flag.CommandLine) pflag.Parse() - addr := "127.0.0.1:10660" - if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" { - addr = util.JoinHostPort(os.Getenv("POD_IP"), 10660) + if *port <= 0 { + klog.Errorf("invalid port: %d", port) + os.Exit(1) } + ip := os.Getenv("POD_IP") + if net.ParseIP(ip) == nil { + klog.Errorf("invalid ip: %q", ip) + os.Exit(1) + } + + addr := util.JoinHostPort(ip, *port) if *tls { addr = "tls://" + addr } else { addr = "tcp://" + addr } - - if err := util.DialTCP(addr, time.Second, false); err != nil { + if err := util.DialTCP(addr, 100*time.Millisecond, false); err != nil { util.LogFatalAndExit(err, "failed to probe the socket") } } diff --git a/dist/images/Dockerfile b/dist/images/Dockerfile index d3ca490cfcf..5e39f82d8a3 100644 --- a/dist/images/Dockerfile +++ b/dist/images/Dockerfile @@ -15,7 +15,7 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-monitor && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-webhook && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \ + ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-healthcheck && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller && \ setcap CAP_NET_BIND_SERVICE+eip /kube-ovn/kube-ovn-cmd && \ diff --git a/dist/images/install.sh b/dist/images/install.sh index 3d864f92ec0..c484a6bba57 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -4341,19 +4341,21 @@ spec: readinessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 - --tls=${SECURE_SERVING} periodSeconds: 3 - timeoutSeconds: 45 + timeoutSeconds: 1 livenessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 - --tls=${SECURE_SERVING} initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 - timeoutSeconds: 45 + timeoutSeconds: 1 resources: requests: cpu: 200m @@ -4561,16 +4563,22 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls=${SECURE_SERVING} + timeoutSeconds: 1 readinessProbe: failureThreshold: 3 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls=${SECURE_SERVING} + timeoutSeconds: 1 resources: requests: cpu: 100m @@ -4896,17 +4904,23 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10661 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls=${SECURE_SERVING} + timeoutSeconds: 1 readinessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10661 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls=${SECURE_SERVING} + timeoutSeconds: 1 nodeSelector: kubernetes.io/os: "linux" kube-ovn/role: "master" diff --git a/pkg/server/server.go b/pkg/server/server.go deleted file mode 100644 index e3041693974..00000000000 --- a/pkg/server/server.go +++ /dev/null @@ -1,101 +0,0 @@ -package server - -import ( - "fmt" - "net" - "net/http" - "os" - "strconv" - "strings" - - "k8s.io/apiserver/pkg/endpoints/filters" - "k8s.io/apiserver/pkg/endpoints/request" - "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/options" - "k8s.io/client-go/rest" - "k8s.io/klog/v2" - - "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" -) - -func SecureServing(addr, svcName string, handler http.Handler) (<-chan struct{}, error) { - host, port, err := net.SplitHostPort(addr) - if err != nil { - klog.Error(err) - return nil, fmt.Errorf("invalid listen address %q: %w", addr, err) - } - - namespace := os.Getenv("POD_NAMESPACE") - podName := os.Getenv("POD_NAME") - podIPs := os.Getenv("POD_IPS") - alternateDNS := []string{podName, svcName, fmt.Sprintf("%s.%s", svcName, namespace), fmt.Sprintf("%s.%s.svc", svcName, namespace)} - alternateIPs := []net.IP{net.ParseIP("127.0.0.1"), net.IPv6loopback} - for _, podIP := range strings.Split(podIPs, ",") { - if ip := net.ParseIP(podIP); ip != nil { - alternateIPs = append(alternateIPs, ip) - } - } - - var clientConfig *rest.Config - opt := options.NewSecureServingOptions().WithLoopback() - authnOpt := options.NewDelegatingAuthenticationOptions() - authzOpt := options.NewDelegatingAuthorizationOptions() - opt.ServerCert.PairName = svcName - opt.ServerCert.CertDirectory = "" - authnOpt.RemoteKubeConfigFileOptional = true - authzOpt.RemoteKubeConfigFileOptional = true - - if host != "" { - ip := net.ParseIP(host) - if ip == nil { - err = fmt.Errorf("invalid listen address: %q", addr) - klog.Error(err) - return nil, err - } - opt.BindAddress = ip - p, err := strconv.Atoi(port) - if err != nil { - klog.Error(err) - return nil, fmt.Errorf("invalid listen address %q: %w", addr, err) - } - opt.BindPort = p - } - - if err = opt.MaybeDefaultWithSelfSignedCerts("localhost", alternateDNS, alternateIPs); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to generate self signed certificates: %w", err) - } - - var serving *server.SecureServingInfo - var authn server.AuthenticationInfo - var authz server.AuthorizationInfo - if err = opt.ApplyTo(&serving, &clientConfig); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to apply secure serving options to secure serving info: %w", err) - } - if err = authnOpt.ApplyTo(&authn, serving, nil); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to apply authn options to authn info: %w", err) - } - if err = authzOpt.ApplyTo(&authz); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to apply authz options to authz info: %w", err) - } - - handler = filters.WithAuthorization(handler, authz.Authorizer, scheme.Codecs) - handler = filters.WithAuthentication(handler, authn.Authenticator, filters.Unauthorized(scheme.Codecs), nil, nil) - - requestInfoResolver := &request.RequestInfoFactory{} - handler = filters.WithRequestInfo(handler, requestInfoResolver) - handler = filters.WithCacheControl(handler) - server.AuthorizeClientBearerToken(clientConfig, &authn, &authz) - - stopCh := make(chan struct{}, 1) - _, listenerStoppedCh, err := serving.Serve(handler, 0, stopCh) - if err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to serve on %s: %w", addr, err) - } - - return listenerStoppedCh, nil -} From 3fc077246a4dc069f2cbd29e3991f076ba764f3f Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Mon, 12 Aug 2024 02:22:35 +0000 Subject: [PATCH 2/2] go mod tidy Signed-off-by: zhangzujian --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index afe6e6ddd47..0d45743bce8 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,6 @@ require ( gopkg.in/k8snetworkplumbingwg/multus-cni.v4 v4.1.0 k8s.io/api v0.30.3 k8s.io/apimachinery v0.30.3 - k8s.io/apiserver v0.30.3 k8s.io/client-go v12.0.0+incompatible k8s.io/klog/v2 v2.130.1 k8s.io/kubectl v0.30.3 @@ -248,6 +247,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.3 // indirect + k8s.io/apiserver v0.30.3 // indirect k8s.io/cli-runtime v0.30.3 // indirect k8s.io/cloud-provider v0.30.3 // indirect k8s.io/cluster-bootstrap v0.30.3 // indirect