diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml index 5c43dd6a178..fa91bbaeda4 100644 --- a/charts/kube-ovn/templates/controller-deploy.yaml +++ b/charts/kube-ovn/templates/controller-deploy.yaml @@ -168,19 +168,21 @@ spec: readinessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 - --tls={{- .Values.func.SECURE_SERVING }} periodSeconds: 3 - timeoutSeconds: 45 + timeoutSeconds: 1 livenessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 - --tls={{- .Values.func.SECURE_SERVING }} initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 - timeoutSeconds: 45 + timeoutSeconds: 1 resources: requests: cpu: {{ index .Values "kube-ovn-controller" "requests" "cpu" }} diff --git a/charts/kube-ovn/templates/monitor-deploy.yaml b/charts/kube-ovn/templates/monitor-deploy.yaml index 88bf77c2985..befd8181d12 100644 --- a/charts/kube-ovn/templates/monitor-deploy.yaml +++ b/charts/kube-ovn/templates/monitor-deploy.yaml @@ -108,17 +108,23 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10661 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 1 readinessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10661 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 1 nodeSelector: kubernetes.io/os: "linux" {{- with splitList "=" .Values.MASTER_NODES_LABEL }} diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml index 7bc6a81c52e..47c60efb80c 100644 --- a/charts/kube-ovn/templates/ovncni-ds.yaml +++ b/charts/kube-ovn/templates/ovncni-ds.yaml @@ -148,17 +148,23 @@ spec: failureThreshold: 3 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 1 livenessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls={{- .Values.func.SECURE_SERVING }} + timeoutSeconds: 1 resources: requests: cpu: {{ index .Values "kube-ovn-cni" "requests" "cpu" }} diff --git a/cmd/cmdmain.go b/cmd/cmdmain.go index d10a1ddc74d..be92453a35a 100644 --- a/cmd/cmdmain.go +++ b/cmd/cmdmain.go @@ -12,8 +12,8 @@ import ( "k8s.io/klog/v2" "github.com/kubeovn/kube-ovn/cmd/controller" - "github.com/kubeovn/kube-ovn/cmd/controller_health_check" "github.com/kubeovn/kube-ovn/cmd/daemon" + "github.com/kubeovn/kube-ovn/cmd/health_check" "github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker" "github.com/kubeovn/kube-ovn/cmd/ovn_monitor" "github.com/kubeovn/kube-ovn/cmd/pinger" @@ -23,14 +23,14 @@ import ( ) const ( - CmdController = "kube-ovn-controller" - CmdDaemon = "kube-ovn-daemon" - CmdMonitor = "kube-ovn-monitor" - CmdPinger = "kube-ovn-pinger" - CmdSpeaker = "kube-ovn-speaker" - CmdWebhook = "kube-ovn-webhook" - CmdControllerHealthCheck = "kube-ovn-controller-healthcheck" - CmdOvnLeaderChecker = "kube-ovn-leader-checker" + CmdController = "kube-ovn-controller" + CmdDaemon = "kube-ovn-daemon" + CmdMonitor = "kube-ovn-monitor" + CmdPinger = "kube-ovn-pinger" + CmdSpeaker = "kube-ovn-speaker" + CmdWebhook = "kube-ovn-webhook" + CmdHealthCheck = "kube-ovn-healthcheck" + CmdOvnLeaderChecker = "kube-ovn-leader-checker" ) const timeFormat = "2006-01-02_15:04:05" @@ -110,8 +110,8 @@ func main() { speaker.CmdMain() case CmdWebhook: webhook.CmdMain() - case CmdControllerHealthCheck: - controller_health_check.CmdMain() + case CmdHealthCheck: + health_check.CmdMain() case CmdOvnLeaderChecker: ovn_leader_checker.CmdMain() default: diff --git a/cmd/controller_health_check/controller_health_check.go b/cmd/health_check/health_check.go similarity index 63% rename from cmd/controller_health_check/controller_health_check.go rename to cmd/health_check/health_check.go index 7e04769210f..45e59382768 100644 --- a/cmd/controller_health_check/controller_health_check.go +++ b/cmd/health_check/health_check.go @@ -1,7 +1,8 @@ -package controller_health_check +package health_check import ( "flag" + "net" "os" "time" @@ -12,7 +13,8 @@ import ( ) func CmdMain() { - tls := pflag.Bool("tls", false, "Whether kube-ovn-controller uses TLS") + port := pflag.Int32("port", 0, "Target port") + tls := pflag.Bool("tls", false, "Dial the server with TLS") klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) klog.InitFlags(klogFlags) @@ -32,18 +34,24 @@ func CmdMain() { pflag.CommandLine.AddGoFlagSet(flag.CommandLine) pflag.Parse() - addr := "127.0.0.1:10660" - if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" { - addr = util.JoinHostPort(os.Getenv("POD_IP"), 10660) + if *port <= 0 { + klog.Errorf("invalid port: %d", port) + os.Exit(1) } + ip := os.Getenv("POD_IP") + if net.ParseIP(ip) == nil { + klog.Errorf("invalid ip: %q", ip) + os.Exit(1) + } + + addr := util.JoinHostPort(ip, *port) if *tls { addr = "tls://" + addr } else { addr = "tcp://" + addr } - - if err := util.DialTCP(addr, time.Second, false); err != nil { + if err := util.DialTCP(addr, 100*time.Millisecond, false); err != nil { util.LogFatalAndExit(err, "failed to probe the socket") } } diff --git a/dist/images/Dockerfile b/dist/images/Dockerfile index 555869bd2f0..db41f2cf039 100644 --- a/dist/images/Dockerfile +++ b/dist/images/Dockerfile @@ -15,7 +15,7 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-webhook && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \ + ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-healthcheck && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker FROM kubeovn/kube-ovn-base:$BASE_TAG diff --git a/dist/images/install.sh b/dist/images/install.sh index fd244dab125..e866aaf7bd0 100644 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -4232,19 +4232,21 @@ spec: readinessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 - --tls=${SECURE_SERVING} periodSeconds: 3 - timeoutSeconds: 45 + timeoutSeconds: 1 livenessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 - --tls=${SECURE_SERVING} initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 - timeoutSeconds: 45 + timeoutSeconds: 1 resources: requests: cpu: 200m @@ -4409,16 +4411,22 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls=${SECURE_SERVING} + timeoutSeconds: 1 readinessProbe: failureThreshold: 3 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls=${SECURE_SERVING} + timeoutSeconds: 1 resources: requests: cpu: 100m @@ -4698,17 +4706,23 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10661 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls=${SECURE_SERVING} + timeoutSeconds: 1 readinessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10661 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls=${SECURE_SERVING} + timeoutSeconds: 1 nodeSelector: kubernetes.io/os: "linux" kube-ovn/role: "master" diff --git a/go.mod b/go.mod index c3581796f1f..944d82a9866 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,6 @@ require ( gopkg.in/k8snetworkplumbingwg/multus-cni.v4 v4.1.0 k8s.io/api v0.30.3 k8s.io/apimachinery v0.30.3 - k8s.io/apiserver v0.30.3 k8s.io/client-go v12.0.0+incompatible k8s.io/klog/v2 v2.130.1 k8s.io/kubectl v0.30.3 @@ -240,6 +239,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.3 // indirect + k8s.io/apiserver v0.30.3 // indirect k8s.io/cli-runtime v0.30.3 // indirect k8s.io/cloud-provider v0.30.3 // indirect k8s.io/cluster-bootstrap v0.30.3 // indirect diff --git a/pkg/server/server.go b/pkg/server/server.go deleted file mode 100644 index e3041693974..00000000000 --- a/pkg/server/server.go +++ /dev/null @@ -1,101 +0,0 @@ -package server - -import ( - "fmt" - "net" - "net/http" - "os" - "strconv" - "strings" - - "k8s.io/apiserver/pkg/endpoints/filters" - "k8s.io/apiserver/pkg/endpoints/request" - "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/options" - "k8s.io/client-go/rest" - "k8s.io/klog/v2" - - "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" -) - -func SecureServing(addr, svcName string, handler http.Handler) (<-chan struct{}, error) { - host, port, err := net.SplitHostPort(addr) - if err != nil { - klog.Error(err) - return nil, fmt.Errorf("invalid listen address %q: %w", addr, err) - } - - namespace := os.Getenv("POD_NAMESPACE") - podName := os.Getenv("POD_NAME") - podIPs := os.Getenv("POD_IPS") - alternateDNS := []string{podName, svcName, fmt.Sprintf("%s.%s", svcName, namespace), fmt.Sprintf("%s.%s.svc", svcName, namespace)} - alternateIPs := []net.IP{net.ParseIP("127.0.0.1"), net.IPv6loopback} - for _, podIP := range strings.Split(podIPs, ",") { - if ip := net.ParseIP(podIP); ip != nil { - alternateIPs = append(alternateIPs, ip) - } - } - - var clientConfig *rest.Config - opt := options.NewSecureServingOptions().WithLoopback() - authnOpt := options.NewDelegatingAuthenticationOptions() - authzOpt := options.NewDelegatingAuthorizationOptions() - opt.ServerCert.PairName = svcName - opt.ServerCert.CertDirectory = "" - authnOpt.RemoteKubeConfigFileOptional = true - authzOpt.RemoteKubeConfigFileOptional = true - - if host != "" { - ip := net.ParseIP(host) - if ip == nil { - err = fmt.Errorf("invalid listen address: %q", addr) - klog.Error(err) - return nil, err - } - opt.BindAddress = ip - p, err := strconv.Atoi(port) - if err != nil { - klog.Error(err) - return nil, fmt.Errorf("invalid listen address %q: %w", addr, err) - } - opt.BindPort = p - } - - if err = opt.MaybeDefaultWithSelfSignedCerts("localhost", alternateDNS, alternateIPs); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to generate self signed certificates: %w", err) - } - - var serving *server.SecureServingInfo - var authn server.AuthenticationInfo - var authz server.AuthorizationInfo - if err = opt.ApplyTo(&serving, &clientConfig); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to apply secure serving options to secure serving info: %w", err) - } - if err = authnOpt.ApplyTo(&authn, serving, nil); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to apply authn options to authn info: %w", err) - } - if err = authzOpt.ApplyTo(&authz); err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to apply authz options to authz info: %w", err) - } - - handler = filters.WithAuthorization(handler, authz.Authorizer, scheme.Codecs) - handler = filters.WithAuthentication(handler, authn.Authenticator, filters.Unauthorized(scheme.Codecs), nil, nil) - - requestInfoResolver := &request.RequestInfoFactory{} - handler = filters.WithRequestInfo(handler, requestInfoResolver) - handler = filters.WithCacheControl(handler) - server.AuthorizeClientBearerToken(clientConfig, &authn, &authz) - - stopCh := make(chan struct{}, 1) - _, listenerStoppedCh, err := serving.Serve(handler, 0, stopCh) - if err != nil { - klog.Error(err) - return nil, fmt.Errorf("failed to serve on %s: %w", addr, err) - } - - return listenerStoppedCh, nil -}