Skip to content

Commit

Permalink
feat: ko support cluster operations status/kick/backup
Browse files Browse the repository at this point in the history
  • Loading branch information
@oilbeater
oilbeater committed Jul 5, 2021
1 parent 9cc0738 commit bdf9184
Show file tree
Hide file tree
Showing 4 changed files with 235 additions and 8 deletions.
79 changes: 75 additions & 4 deletions dist/images/install-pre-1.16.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2069,6 +2069,7 @@ OVN_SB_POD=
showHelp(){
echo "kubectl ko {subcommand} [option...]"
echo "Available Subcommands:"
echo " [nb|sb] [status|kick|backup] ovn-db operations show cluster status, kick stale server or backup database"
echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl"
echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl"
echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on selected node"
Expand Down Expand Up @@ -2111,7 +2112,6 @@ tcpdump(){
echo "nic doesn't exist on node $nodeName"
exit 1
fi
podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type})
podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r')
set -x
Expand Down Expand Up @@ -2220,6 +2220,22 @@ xxctl(){
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@"
}
checkLeader(){
component="$1"; shift
count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l)
if [ $count -eq 0 ]; then
echo "no ovn-$component exists !!"
exit 1
fi
if [ $count -gt 1 ]; then
echo "ovn-$component has more than one leader !!"
exit 1
fi
echo "ovn-$component leader check ok"
}
diagnose(){
kubectl get crd vpcs.kubeovn.io
kubectl get crd vpc-nat-gateways.kubeovn.io
Expand All @@ -2245,6 +2261,11 @@ diagnose(){
checkDaemonSet kube-ovn-cni
checkDaemonSet ovs-ovn
checkDeployment coredns
checkLeader nb
checkLeader sb
checkLeader northd
type="$1"
case $type in
all)
Expand Down Expand Up @@ -2341,19 +2362,65 @@ checkDeployment(){
checkKubeProxy(){
dsMode=`kubectl get ds -n kube-system | grep kube-proxy || true`
if [ -z "$dsMode" ]; then
nodeIps=`kubectl get node -o wide --no-headers | awk '{print $6}'`
nodeIps=`kubectl get node -o wide | grep -v "INTERNAL-IP" | awk '{print $6}'`
for node in $nodeIps
do
healthResult=`curl -g -6 -sL --connect-timeout 5 -w %{http_code} http://[$node]:10256/healthz -o /dev/null | grep -v 200 || true`
healthResult=`curl -g -6 -sL -w %{http_code} http://[$node]:10256/healthz -o /dev/null | grep -v 200 || true`
if [ -n "$healthResult" ]; then
echo "$node kube-proxy's health check failed"
exit 1
fi
done
echo "kube-proxy ready"
else
checkDaemonSet kube-proxy
fi
echo "kube-proxy ready"
}
dbtool(){
suffix=$(date +%m%d%H%M%s)
component="$1"; shift
action="$1"; shift
case $component in
nb)
case $action in
status)
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
;;
kick)
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1"
;;
backup)
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db
kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/etc/ovn/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnnb_db.$suffix.backup
echo "backup $component to $(pwd)/ovnnb_db.$suffix.backup"
;;
*)
echo "unknown action $action"
esac
;;
sb)
case $action in
status)
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound
;;
kick)
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1"
;;
backup)
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db
kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/etc/ovn/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnsb_db.$suffix.backup
echo "backup $component to $(pwd)/ovnsb_db.$suffix.backup"
;;
*)
echo "unknown action $action"
esac
;;
*)
echo "unknown subcommand $component"
esac
}
if [ $# -lt 1 ]; then
Expand All @@ -2375,6 +2442,9 @@ case $subcommand in
vsctl|ofctl|dpctl|appctl)
xxctl "$subcommand" "$@"
;;
nb|sb)
dbtool "$subcommand" "$@"
;;
tcpdump)
tcpdump "$@"
;;
Expand All @@ -2388,6 +2458,7 @@ case $subcommand in
showHelp
;;
esac
EOF

chmod +x /usr/local/bin/kubectl-ko
Expand Down
79 changes: 75 additions & 4 deletions dist/images/install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2107,6 +2107,7 @@ OVN_SB_POD=
showHelp(){
echo "kubectl ko {subcommand} [option...]"
echo "Available Subcommands:"
echo " [nb|sb] [status|kick|backup] ovn-db operations show cluster status, kick stale server or backup database"
echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl"
echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl"
echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on selected node"
Expand Down Expand Up @@ -2149,7 +2150,6 @@ tcpdump(){
echo "nic doesn't exist on node $nodeName"
exit 1
fi
podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type})
podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r')
set -x
Expand Down Expand Up @@ -2258,6 +2258,22 @@ xxctl(){
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@"
}
checkLeader(){
component="$1"; shift
count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l)
if [ $count -eq 0 ]; then
echo "no ovn-$component exists !!"
exit 1
fi
if [ $count -gt 1 ]; then
echo "ovn-$component has more than one leader !!"
exit 1
fi
echo "ovn-$component leader check ok"
}
diagnose(){
kubectl get crd vpcs.kubeovn.io
kubectl get crd vpc-nat-gateways.kubeovn.io
Expand All @@ -2283,6 +2299,11 @@ diagnose(){
checkDaemonSet kube-ovn-cni
checkDaemonSet ovs-ovn
checkDeployment coredns
checkLeader nb
checkLeader sb
checkLeader northd
type="$1"
case $type in
all)
Expand Down Expand Up @@ -2379,19 +2400,65 @@ checkDeployment(){
checkKubeProxy(){
dsMode=`kubectl get ds -n kube-system | grep kube-proxy || true`
if [ -z "$dsMode" ]; then
nodeIps=`kubectl get node -o wide --no-headers | awk '{print $6}'`
nodeIps=`kubectl get node -o wide | grep -v "INTERNAL-IP" | awk '{print $6}'`
for node in $nodeIps
do
healthResult=`curl -g -6 -sL --connect-timeout 5 -w %{http_code} http://[$node]:10256/healthz -o /dev/null | grep -v 200 || true`
healthResult=`curl -g -6 -sL -w %{http_code} http://[$node]:10256/healthz -o /dev/null | grep -v 200 || true`
if [ -n "$healthResult" ]; then
echo "$node kube-proxy's health check failed"
exit 1
fi
done
echo "kube-proxy ready"
else
checkDaemonSet kube-proxy
fi
echo "kube-proxy ready"
}
dbtool(){
suffix=$(date +%m%d%H%M%s)
component="$1"; shift
action="$1"; shift
case $component in
nb)
case $action in
status)
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
;;
kick)
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1"
;;
backup)
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db
kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/etc/ovn/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnnb_db.$suffix.backup
echo "backup $component to $(pwd)/ovnnb_db.$suffix.backup"
;;
*)
echo "unknown action $action"
esac
;;
sb)
case $action in
status)
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound
;;
kick)
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1"
;;
backup)
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db
kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/etc/ovn/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnsb_db.$suffix.backup
echo "backup $component to $(pwd)/ovnsb_db.$suffix.backup"
;;
*)
echo "unknown action $action"
esac
;;
*)
echo "unknown subcommand $component"
esac
}
if [ $# -lt 1 ]; then
Expand All @@ -2413,6 +2480,9 @@ case $subcommand in
vsctl|ofctl|dpctl|appctl)
xxctl "$subcommand" "$@"
;;
nb|sb)
dbtool "$subcommand" "$@"
;;
tcpdump)
tcpdump "$@"
;;
Expand All @@ -2426,6 +2496,7 @@ case $subcommand in
showHelp
;;
esac
EOF

chmod +x /usr/local/bin/kubectl-ko
Expand Down
71 changes: 71 additions & 0 deletions dist/images/kubectl-ko
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ OVN_SB_POD=
showHelp(){
echo "kubectl ko {subcommand} [option...]"
echo "Available Subcommands:"
echo " [nb|sb] [status|kick|backup] ovn-db operations show cluster status, kick stale server or backup database"
echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl"
echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl"
echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on selected node"
Expand Down Expand Up @@ -158,6 +159,22 @@ xxctl(){
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@"
}

checkLeader(){
component="$1"; shift
count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l)
if [ $count -eq 0 ]; then
echo "no ovn-$component exists !!"
exit 1
fi

if [ $count -gt 1 ]; then
echo "ovn-$component has more than one leader !!"
exit 1
fi

echo "ovn-$component leader check ok"
}

diagnose(){
kubectl get crd vpcs.kubeovn.io
kubectl get crd vpc-nat-gateways.kubeovn.io
Expand All @@ -183,6 +200,11 @@ diagnose(){
checkDaemonSet kube-ovn-cni
checkDaemonSet ovs-ovn
checkDeployment coredns

checkLeader nb
checkLeader sb
checkLeader northd

type="$1"
case $type in
all)
Expand Down Expand Up @@ -294,6 +316,52 @@ checkKubeProxy(){
echo "kube-proxy ready"
}

dbtool(){
suffix=$(date +%m%d%H%M%s)
component="$1"; shift
action="$1"; shift
case $component in
nb)
case $action in
status)
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
;;
kick)
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1"
;;
backup)
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /tmp/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db
kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/tmp/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /tmp/ovnnb_db.$suffix.backup
echo "backup ovn-$component db to $(pwd)/ovnnb_db.$suffix.backup"
;;
*)
echo "unknown action $action"
esac
;;
sb)
case $action in
status)
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound
;;
kick)
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1"
;;
backup)
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /tmp/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db
kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/tmp/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /tmp/ovnsb_db.$suffix.backup
echo "backup ovn-$component db to $(pwd)/ovnsb_db.$suffix.backup"
;;
*)
echo "unknown action $action"
esac
;;
*)
echo "unknown subcommand $component"
esac
}

if [ $# -lt 1 ]; then
showHelp
exit 0
Expand All @@ -313,6 +381,9 @@ case $subcommand in
vsctl|ofctl|dpctl|appctl)
xxctl "$subcommand" "$@"
;;
nb|sb)
dbtool "$subcommand" "$@"
;;
tcpdump)
tcpdump "$@"
;;
Expand Down
Loading

0 comments on commit bdf9184

Please sign in to comment.