fix EOF during TLS handshake caused by health check (kubeovn#4381)

Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>
zcq98 · Aug 12, 2024 · fd433cb · fd433cb
1 parent cd98da2
commit fd433cb
Show file tree

Hide file tree

Showing 9 changed files with 87 additions and 152 deletions.
diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml
@@ -187,19 +187,21 @@ spec:
           readinessProbe:
             exec:
               command:
-                - /kube-ovn/kube-ovn-controller-healthcheck
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10660
                 - --tls={{- .Values.func.SECURE_SERVING }}
             periodSeconds: 3
-            timeoutSeconds: 45
+            timeoutSeconds: 1
           livenessProbe:
             exec:
               command:
-                - /kube-ovn/kube-ovn-controller-healthcheck
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10660
                 - --tls={{- .Values.func.SECURE_SERVING }}
             initialDelaySeconds: 300
             periodSeconds: 7
             failureThreshold: 5
-            timeoutSeconds: 45
+            timeoutSeconds: 1
           resources:
             requests:
               cpu: {{ index .Values "kube-ovn-controller" "requests" "cpu" }}

diff --git a/charts/kube-ovn/templates/monitor-deploy.yaml b/charts/kube-ovn/templates/monitor-deploy.yaml
@@ -125,17 +125,23 @@ spec:
             initialDelaySeconds: 30
             periodSeconds: 7
             successThreshold: 1
-            tcpSocket:
-              port: 10661
-            timeoutSeconds: 3
+            exec:
+              command:
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10661
+                - --tls={{- .Values.func.SECURE_SERVING }}
+            timeoutSeconds: 1
           readinessProbe:
             failureThreshold: 3
             initialDelaySeconds: 30
             periodSeconds: 7
             successThreshold: 1
-            tcpSocket:
-              port: 10661
-            timeoutSeconds: 3
+            exec:
+              command:
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10661
+                - --tls={{- .Values.func.SECURE_SERVING }}
+            timeoutSeconds: 1
       nodeSelector:
         kubernetes.io/os: "linux"
         {{- with splitList "=" .Values.MASTER_NODES_LABEL }}

diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml
@@ -198,17 +198,23 @@ spec:
           failureThreshold: 3
           periodSeconds: 7
           successThreshold: 1
-          tcpSocket:
-            port: 10665
-          timeoutSeconds: 3
+          exec:
+            command:
+              - /kube-ovn/kube-ovn-healthcheck
+              - --port=10665
+              - --tls={{- .Values.func.SECURE_SERVING }}
+          timeoutSeconds: 1
         livenessProbe:
           failureThreshold: 3
           initialDelaySeconds: 30
           periodSeconds: 7
           successThreshold: 1
-          tcpSocket:
-            port: 10665
-          timeoutSeconds: 3
+          exec:
+            command:
+              - /kube-ovn/kube-ovn-healthcheck
+              - --port=10665
+              - --tls={{- .Values.func.SECURE_SERVING }}
+          timeoutSeconds: 1
         resources:
           requests:
             cpu: {{ index .Values "kube-ovn-cni" "requests" "cpu" }}

diff --git a/cmd/cmdmain.go b/cmd/cmdmain.go
@@ -12,7 +12,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"github.com/kubeovn/kube-ovn/cmd/controller"
-	"github.com/kubeovn/kube-ovn/cmd/controller_health_check"
+	"github.com/kubeovn/kube-ovn/cmd/health_check"
 	"github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller"
 	"github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker"
 	"github.com/kubeovn/kube-ovn/cmd/ovn_monitor"
@@ -22,13 +22,13 @@ import (
 )
 
 const (
-	CmdController            = "kube-ovn-controller"
-	CmdMonitor               = "kube-ovn-monitor"
-	CmdSpeaker               = "kube-ovn-speaker"
-	CmdWebhook               = "kube-ovn-webhook"
-	CmdControllerHealthCheck = "kube-ovn-controller-healthcheck"
-	CmdOvnLeaderChecker      = "kube-ovn-leader-checker"
-	CmdOvnICController       = "kube-ovn-ic-controller"
+	CmdController       = "kube-ovn-controller"
+	CmdMonitor          = "kube-ovn-monitor"
+	CmdSpeaker          = "kube-ovn-speaker"
+	CmdWebhook          = "kube-ovn-webhook"
+	CmdHealthCheck      = "kube-ovn-healthcheck"
+	CmdOvnLeaderChecker = "kube-ovn-leader-checker"
+	CmdOvnICController  = "kube-ovn-ic-controller"
 )
 
 const timeFormat = "2006-01-02_15:04:05"
@@ -102,8 +102,8 @@ func main() {
 		speaker.CmdMain()
 	case CmdWebhook:
 		webhook.CmdMain()
-	case CmdControllerHealthCheck:
-		controller_health_check.CmdMain()
+	case CmdHealthCheck:
+		health_check.CmdMain()
 	case CmdOvnLeaderChecker:
 		ovn_leader_checker.CmdMain()
 	case CmdOvnICController:

diff --git a/...r_health_check/controller_health_check.go → cmd/health_check/health_check.go b/...r_health_check/controller_health_check.go → cmd/health_check/health_check.go
@@ -1,7 +1,8 @@
-package controller_health_check
+package health_check
 
 import (
 	"flag"
+	"net"
 	"os"
 	"time"
 
@@ -12,7 +13,8 @@ import (
 )
 
 func CmdMain() {
-	tls := pflag.Bool("tls", false, "Whether kube-ovn-controller uses TLS")
+	port := pflag.Int32("port", 0, "Target port")
+	tls := pflag.Bool("tls", false, "Dial the server with TLS")
 
 	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(klogFlags)
@@ -32,18 +34,24 @@ func CmdMain() {
 	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
 	pflag.Parse()
 
-	addr := "127.0.0.1:10660"
-	if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
-		addr = util.JoinHostPort(os.Getenv("POD_IP"), 10660)
+	if *port <= 0 {
+		klog.Errorf("invalid port: %d", port)
+		os.Exit(1)
 	}
 
+	ip := os.Getenv("POD_IP")
+	if net.ParseIP(ip) == nil {
+		klog.Errorf("invalid ip: %q", ip)
+		os.Exit(1)
+	}
+
+	addr := util.JoinHostPort(ip, *port)
 	if *tls {
 		addr = "tls://" + addr
 	} else {
 		addr = "tcp://" + addr
 	}
-
-	if err := util.DialTCP(addr, time.Second, false); err != nil {
+	if err := util.DialTCP(addr, 100*time.Millisecond, false); err != nil {
 		util.LogFatalAndExit(err, "failed to probe the socket")
 	}
 }
diff --git a/dist/images/Dockerfile b/dist/images/Dockerfile
@@ -15,7 +15,7 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \
     ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-monitor && \
     ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \
     ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-webhook && \
-    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \
+    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-healthcheck && \
     ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \
     ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller && \
     setcap CAP_NET_BIND_SERVICE+eip /kube-ovn/kube-ovn-cmd && \

diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -4341,19 +4341,21 @@ spec:
           readinessProbe:
             exec:
               command:
-                - /kube-ovn/kube-ovn-controller-healthcheck
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10660
                 - --tls=${SECURE_SERVING}
             periodSeconds: 3
-            timeoutSeconds: 45
+            timeoutSeconds: 1
           livenessProbe:
             exec:
               command:
-                - /kube-ovn/kube-ovn-controller-healthcheck
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10660
                 - --tls=${SECURE_SERVING}
             initialDelaySeconds: 300
             periodSeconds: 7
             failureThreshold: 5
-            timeoutSeconds: 45
+            timeoutSeconds: 1
           resources:
             requests:
               cpu: 200m
@@ -4561,16 +4563,22 @@ spec:
           initialDelaySeconds: 30
           periodSeconds: 7
           successThreshold: 1
-          tcpSocket:
-            port: 10665
-          timeoutSeconds: 3
+          exec:
+            command:
+              - /kube-ovn/kube-ovn-healthcheck
+              - --port=10665
+              - --tls=${SECURE_SERVING}
+          timeoutSeconds: 1
         readinessProbe:
           failureThreshold: 3
           periodSeconds: 7
           successThreshold: 1
-          tcpSocket:
-            port: 10665
-          timeoutSeconds: 3
+          exec:
+            command:
+              - /kube-ovn/kube-ovn-healthcheck
+              - --port=10665
+              - --tls=${SECURE_SERVING}
+          timeoutSeconds: 1
         resources:
           requests:
             cpu: 100m
@@ -4896,17 +4904,23 @@ spec:
             initialDelaySeconds: 30
             periodSeconds: 7
             successThreshold: 1
-            tcpSocket:
-              port: 10661
-            timeoutSeconds: 3
+            exec:
+              command:
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10661
+                - --tls=${SECURE_SERVING}
+            timeoutSeconds: 1
           readinessProbe:
             failureThreshold: 3
             initialDelaySeconds: 30
             periodSeconds: 7
             successThreshold: 1
-            tcpSocket:
-              port: 10661
-            timeoutSeconds: 3
+            exec:
+              command:
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10661
+                - --tls=${SECURE_SERVING}
+            timeoutSeconds: 1
       nodeSelector:
         kubernetes.io/os: "linux"
         kube-ovn/role: "master"

diff --git a/go.mod b/go.mod
@@ -45,7 +45,6 @@ require (
 	gopkg.in/k8snetworkplumbingwg/multus-cni.v4 v4.1.0
 	k8s.io/api v0.30.3
 	k8s.io/apimachinery v0.30.3
-	k8s.io/apiserver v0.30.3
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/kubectl v0.30.3
@@ -248,6 +247,7 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.30.3 // indirect
+	k8s.io/apiserver v0.30.3 // indirect
 	k8s.io/cli-runtime v0.30.3 // indirect
 	k8s.io/cloud-provider v0.30.3 // indirect
 	k8s.io/cluster-bootstrap v0.30.3 // indirect

diff --git a/pkg/server/server.go b/pkg/server/server.go