diff --git a/dist/images/install-pre-1.16.sh b/dist/images/install-pre-1.16.sh index fb55317c71c..c6ed7c14932 100644 --- a/dist/images/install-pre-1.16.sh +++ b/dist/images/install-pre-1.16.sh @@ -2069,6 +2069,7 @@ OVN_SB_POD= showHelp(){ echo "kubectl ko {subcommand} [option...]" echo "Available Subcommands:" + echo " [nb|sb] [status|kick|backup] ovn-db operations show cluster status, kick stale server or backup database" echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl" echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl" echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on selected node" @@ -2111,7 +2112,6 @@ tcpdump(){ echo "nic doesn't exist on node $nodeName" exit 1 fi - podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type}) podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r') set -x @@ -2220,6 +2220,22 @@ xxctl(){ kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@" } +checkLeader(){ + component="$1"; shift + count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l) + if [ $count -eq 0 ]; then + echo "no ovn-$component exists !!" + exit 1 + fi + + if [ $count -gt 1 ]; then + echo "ovn-$component has more than one leader !!" + exit 1 + fi + + echo "ovn-$component leader check ok" +} + diagnose(){ kubectl get crd vpcs.kubeovn.io kubectl get crd vpc-nat-gateways.kubeovn.io @@ -2245,6 +2261,11 @@ diagnose(){ checkDaemonSet kube-ovn-cni checkDaemonSet ovs-ovn checkDeployment coredns + + checkLeader nb + checkLeader sb + checkLeader northd + type="$1" case $type in all) @@ -2341,19 +2362,65 @@ checkDeployment(){ checkKubeProxy(){ dsMode=`kubectl get ds -n kube-system | grep kube-proxy || true` if [ -z "$dsMode" ]; then - nodeIps=`kubectl get node -o wide --no-headers | awk '{print $6}'` + nodeIps=`kubectl get node -o wide | grep -v "INTERNAL-IP" | awk '{print $6}'` for node in $nodeIps do - healthResult=`curl -g -6 -sL --connect-timeout 5 -w %{http_code} http://[$node]:10256/healthz -o /dev/null | grep -v 200 || true` + healthResult=`curl -g -6 -sL -w %{http_code} http://[$node]:10256/healthz -o /dev/null | grep -v 200 || true` if [ -n "$healthResult" ]; then echo "$node kube-proxy's health check failed" exit 1 fi done - echo "kube-proxy ready" else checkDaemonSet kube-proxy fi + echo "kube-proxy ready" +} + +dbtool(){ + suffix=$(date +%m%d%H%M%s) + component="$1"; shift + action="$1"; shift + case $component in + nb) + case $action in + status) + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound + ;; + kick) + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1" + ;; + backup) + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db + kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/etc/ovn/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnnb_db.$suffix.backup + echo "backup $component to $(pwd)/ovnnb_db.$suffix.backup" + ;; + *) + echo "unknown action $action" + esac + ;; + sb) + case $action in + status) + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound + ;; + kick) + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1" + ;; + backup) + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db + kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/etc/ovn/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnsb_db.$suffix.backup + echo "backup $component to $(pwd)/ovnsb_db.$suffix.backup" + ;; + *) + echo "unknown action $action" + esac + ;; + *) + echo "unknown subcommand $component" + esac } if [ $# -lt 1 ]; then @@ -2375,6 +2442,9 @@ case $subcommand in vsctl|ofctl|dpctl|appctl) xxctl "$subcommand" "$@" ;; + nb|sb) + dbtool "$subcommand" "$@" + ;; tcpdump) tcpdump "$@" ;; @@ -2388,6 +2458,7 @@ case $subcommand in showHelp ;; esac + EOF chmod +x /usr/local/bin/kubectl-ko diff --git a/dist/images/install.sh b/dist/images/install.sh index af62b80c52b..2ef1e56bdc2 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -2107,6 +2107,7 @@ OVN_SB_POD= showHelp(){ echo "kubectl ko {subcommand} [option...]" echo "Available Subcommands:" + echo " [nb|sb] [status|kick|backup] ovn-db operations show cluster status, kick stale server or backup database" echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl" echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl" echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on selected node" @@ -2149,7 +2150,6 @@ tcpdump(){ echo "nic doesn't exist on node $nodeName" exit 1 fi - podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type}) podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r') set -x @@ -2258,6 +2258,22 @@ xxctl(){ kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@" } +checkLeader(){ + component="$1"; shift + count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l) + if [ $count -eq 0 ]; then + echo "no ovn-$component exists !!" + exit 1 + fi + + if [ $count -gt 1 ]; then + echo "ovn-$component has more than one leader !!" + exit 1 + fi + + echo "ovn-$component leader check ok" +} + diagnose(){ kubectl get crd vpcs.kubeovn.io kubectl get crd vpc-nat-gateways.kubeovn.io @@ -2283,6 +2299,11 @@ diagnose(){ checkDaemonSet kube-ovn-cni checkDaemonSet ovs-ovn checkDeployment coredns + + checkLeader nb + checkLeader sb + checkLeader northd + type="$1" case $type in all) @@ -2379,19 +2400,65 @@ checkDeployment(){ checkKubeProxy(){ dsMode=`kubectl get ds -n kube-system | grep kube-proxy || true` if [ -z "$dsMode" ]; then - nodeIps=`kubectl get node -o wide --no-headers | awk '{print $6}'` + nodeIps=`kubectl get node -o wide | grep -v "INTERNAL-IP" | awk '{print $6}'` for node in $nodeIps do - healthResult=`curl -g -6 -sL --connect-timeout 5 -w %{http_code} http://[$node]:10256/healthz -o /dev/null | grep -v 200 || true` + healthResult=`curl -g -6 -sL -w %{http_code} http://[$node]:10256/healthz -o /dev/null | grep -v 200 || true` if [ -n "$healthResult" ]; then echo "$node kube-proxy's health check failed" exit 1 fi done - echo "kube-proxy ready" else checkDaemonSet kube-proxy fi + echo "kube-proxy ready" +} + +dbtool(){ + suffix=$(date +%m%d%H%M%s) + component="$1"; shift + action="$1"; shift + case $component in + nb) + case $action in + status) + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound + ;; + kick) + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1" + ;; + backup) + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db + kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/etc/ovn/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnnb_db.$suffix.backup + echo "backup $component to $(pwd)/ovnnb_db.$suffix.backup" + ;; + *) + echo "unknown action $action" + esac + ;; + sb) + case $action in + status) + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound + ;; + kick) + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1" + ;; + backup) + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db + kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/etc/ovn/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnsb_db.$suffix.backup + echo "backup $component to $(pwd)/ovnsb_db.$suffix.backup" + ;; + *) + echo "unknown action $action" + esac + ;; + *) + echo "unknown subcommand $component" + esac } if [ $# -lt 1 ]; then @@ -2413,6 +2480,9 @@ case $subcommand in vsctl|ofctl|dpctl|appctl) xxctl "$subcommand" "$@" ;; + nb|sb) + dbtool "$subcommand" "$@" + ;; tcpdump) tcpdump "$@" ;; @@ -2426,6 +2496,7 @@ case $subcommand in showHelp ;; esac + EOF chmod +x /usr/local/bin/kubectl-ko diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko index 452d9857ed5..fad574b259b 100644 --- a/dist/images/kubectl-ko +++ b/dist/images/kubectl-ko @@ -8,6 +8,7 @@ OVN_SB_POD= showHelp(){ echo "kubectl ko {subcommand} [option...]" echo "Available Subcommands:" + echo " [nb|sb] [status|kick|backup] ovn-db operations show cluster status, kick stale server or backup database" echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl" echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl" echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on selected node" @@ -158,6 +159,22 @@ xxctl(){ kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@" } +checkLeader(){ + component="$1"; shift + count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l) + if [ $count -eq 0 ]; then + echo "no ovn-$component exists !!" + exit 1 + fi + + if [ $count -gt 1 ]; then + echo "ovn-$component has more than one leader !!" + exit 1 + fi + + echo "ovn-$component leader check ok" +} + diagnose(){ kubectl get crd vpcs.kubeovn.io kubectl get crd vpc-nat-gateways.kubeovn.io @@ -183,6 +200,11 @@ diagnose(){ checkDaemonSet kube-ovn-cni checkDaemonSet ovs-ovn checkDeployment coredns + + checkLeader nb + checkLeader sb + checkLeader northd + type="$1" case $type in all) @@ -294,6 +316,52 @@ checkKubeProxy(){ echo "kube-proxy ready" } +dbtool(){ + suffix=$(date +%m%d%H%M%s) + component="$1"; shift + action="$1"; shift + case $component in + nb) + case $action in + status) + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound + ;; + kick) + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1" + ;; + backup) + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /tmp/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db + kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/tmp/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup + kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /tmp/ovnnb_db.$suffix.backup + echo "backup ovn-$component db to $(pwd)/ovnnb_db.$suffix.backup" + ;; + *) + echo "unknown action $action" + esac + ;; + sb) + case $action in + status) + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound + ;; + kick) + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1" + ;; + backup) + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /tmp/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db + kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/tmp/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /tmp/ovnsb_db.$suffix.backup + echo "backup ovn-$component db to $(pwd)/ovnsb_db.$suffix.backup" + ;; + *) + echo "unknown action $action" + esac + ;; + *) + echo "unknown subcommand $component" + esac +} + if [ $# -lt 1 ]; then showHelp exit 0 @@ -313,6 +381,9 @@ case $subcommand in vsctl|ofctl|dpctl|appctl) xxctl "$subcommand" "$@" ;; + nb|sb) + dbtool "$subcommand" "$@" + ;; tcpdump) tcpdump "$@" ;; diff --git a/test/e2e/kubectl-ko/ko.go b/test/e2e/kubectl-ko/ko.go index 41f51a9b763..03635a731cd 100644 --- a/test/e2e/kubectl-ko/ko.go +++ b/test/e2e/kubectl-ko/ko.go @@ -82,4 +82,18 @@ var _ = Describe("[kubectl-ko]", func() { output, err = exec.Command("kubectl", "ko", "trace", fmt.Sprintf("kube-system/%s", pod.Name), "114.114.114.114", "udp", "53").CombinedOutput() Expect(err).NotTo(HaveOccurred(), string(output)) }) + + It("nb/sb operation", func() { + output, err := exec.Command("kubectl", "ko", "nb", "status").CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(output)) + + output, err = exec.Command("kubectl", "ko", "sb", "status").CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(output)) + + output, err = exec.Command("kubectl", "ko", "nb", "backup").CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(output)) + + output, err = exec.Command("kubectl", "ko", "sb", "backup").CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(output)) + }) })