Skip to content

Commit

Permalink
iptables: use the same mode with kube-proxy (kubeovn#2535)
Browse files Browse the repository at this point in the history
  • Loading branch information
zhangzujian committed May 6, 2023
1 parent 944f304 commit 3688ab3
Show file tree
Hide file tree
Showing 9 changed files with 453 additions and 103 deletions.
1 change: 1 addition & 0 deletions dist/images/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ COPY grace_stop_ovn_controller /usr/share/ovn/scripts/grace_stop_ovn_controller

WORKDIR /kube-ovn

RUN /kube-ovn/iptables-wrapper-installer.sh --no-sanity-check
RUN rm -f /usr/bin/nc &&\
rm -f /usr/bin/netcat
RUN deluser sync
Expand Down
3 changes: 0 additions & 3 deletions dist/images/Dockerfile.base
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,6 @@ RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname
logrotate dnsutils net-tools strongswan strongswan-pki libcharon-extra-plugins \
libcharon-extauth-plugins libstrongswan-extra-plugins libstrongswan-standard-plugins -y --no-install-recommends && \
rm -rf /var/lib/apt/lists/* && \
cd /usr/sbin && \
ln -sf /usr/sbin/iptables-legacy iptables && \
ln -sf /usr/sbin/ip6tables-legacy ip6tables && \
rm -rf /etc/localtime

RUN mkdir -p /var/run/openvswitch && \
Expand Down
211 changes: 211 additions & 0 deletions dist/images/iptables-wrapper-installer.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
#!/bin/sh

# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Usage:
#
# iptables-wrapper-installer.sh [--no-sanity-check]
#
# Installs a wrapper iptables script in a container that will figure out
# whether iptables-legacy or iptables-nft is in use on the host and then
# replaces itself with the correct underlying iptables version.
#
# Unless "--no-sanity-check" is passed, it will first verify that the
# container already contains a suitable version of iptables.

# NOTE: This can only use POSIX /bin/sh features; the build container
# might not contain bash.

# original source:
# https://github.com/kubernetes-sigs/iptables-wrappers/blob/master/iptables-wrapper-installer.sh

set -eu

# Find iptables binary location
if [ -d /usr/sbin -a -e /usr/sbin/iptables ]; then
sbin="/usr/sbin"
elif [ -d /sbin -a -e /sbin/iptables ]; then
sbin="/sbin"
else
echo "ERROR: iptables is not present in either /usr/sbin or /sbin" 1>&2
exit 1
fi

# Determine how the system selects between iptables-legacy and iptables-nft
if [ -x /usr/sbin/alternatives ]; then
# Fedora/SUSE style alternatives
altstyle="fedora"
elif [ -x /usr/sbin/update-alternatives ]; then
# Debian style alternatives
altstyle="debian"
else
# No alternatives system
altstyle="none"
fi

if [ "${1:-}" != "--no-sanity-check" ]; then
# Ensure dependencies are installed
if ! version=$("${sbin}/iptables-nft" --version 2> /dev/null); then
echo "ERROR: iptables-nft is not installed" 1>&2
exit 1
fi
if ! "${sbin}/iptables-legacy" --version > /dev/null 2>&1; then
echo "ERROR: iptables-legacy is not installed" 1>&2
exit 1
fi

case "${version}" in
*v1.8.[0123]\ *)
echo "ERROR: iptables 1.8.0 - 1.8.3 have compatibility bugs." 1>&2
echo " Upgrade to 1.8.4 or newer." 1>&2
exit 1
;;
*)
# 1.8.4+ are OK
;;
esac
fi

# Start creating the wrapper...
rm -f "${sbin}/iptables-wrapper"
cat > "${sbin}/iptables-wrapper" <<EOF
#!/bin/sh
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# NOTE: This can only use POSIX /bin/sh features; the container image
# might not contain bash.
set -eu
# In kubernetes 1.17 and later, kubelet will have created at least
# one chain in the "mangle" table (either "KUBE-IPTABLES-HINT" or
# "KUBE-KUBELET-CANARY"), so check that first, against
# iptables-nft, because we can check that more efficiently and
# it's more common these days.
nft_kubelet_rules=\$( (iptables-nft-save -t mangle || true; ip6tables-nft-save -t mangle || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
if [ "\${nft_kubelet_rules}" -ne 0 ]; then
mode=nft
else
# Check for kubernetes 1.17-or-later with iptables-legacy. We
# can't pass "-t mangle" to iptables-legacy-save because it would
# cause the kernel to create that table if it didn't already
# exist, which we don't want. So we have to grab all the rules
legacy_kubelet_rules=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
if [ "\${legacy_kubelet_rules}" -ne 0 ]; then
mode=legacy
else
# With older kubernetes releases there may not be any _specific_
# rules we can look for, but we assume that some non-containerized process
# (possibly kubelet) will have created _some_ iptables rules.
num_legacy_lines=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l)
num_nft_lines=\$( (iptables-nft-save || true; ip6tables-nft-save || true) 2>/dev/null | grep '^-' | wc -l)
if [ "\${num_legacy_lines}" -gt "\${num_nft_lines}" ]; then
mode=legacy
else
mode=nft
fi
fi
fi
EOF

# Write out the appropriate alternatives-selection commands
case "${altstyle}" in
fedora)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
EOF
;;

debian)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
update-alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
update-alternatives --set ip6tables "/usr/sbin/ip6tables-\${mode}" > /dev/null || failed=1
EOF
;;

*)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
rm -f "${sbin}/\${cmd}"
ln -s "${sbin}/xtables-\${mode}-multi" "${sbin}/\${cmd}"
done 2>/dev/null || failed=1
EOF
;;
esac

# Write out the post-alternatives-selection error checking and final wrap-up
cat >> "${sbin}/iptables-wrapper" <<EOF
if [ "\${failed:-0}" = 1 ]; then
echo "Unable to redirect iptables binaries. (Are you running in an unprivileged pod?)" 1>&2
# fake it, though this will probably also fail if they aren't root
exec "${sbin}/xtables-\${mode}-multi" "\$0" "\$@"
fi
# Now re-exec the original command with the newly-selected alternative
exec "\$0" "\$@"
EOF
chmod +x "${sbin}/iptables-wrapper"

# Now back in the installer script, point the iptables binaries at our
# wrapper
case "${altstyle}" in
fedora)
alternatives \
--install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables iptables /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-save iptables-save /usr/sbin/iptables-wrapper
;;

debian)
update-alternatives \
--install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper
update-alternatives \
--install /usr/sbin/ip6tables ip6tables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/ip6tables-restore ip6tables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-save ip6tables-save /usr/sbin/iptables-wrapper
;;

*)
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
rm -f "${sbin}/${cmd}"
ln -s "${sbin}/iptables-wrapper" "${sbin}/${cmd}"
done
;;
esac

# Cleanup
rm -f "$0"
3 changes: 3 additions & 0 deletions dist/images/start-cniserver.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ do
fi
done

# update links to point to the iptables binaries
iptables -V

# If nftables not exist do not exit
set +e
iptables -P FORWARD ACCEPT
Expand Down
3 changes: 3 additions & 0 deletions dist/images/start-ovs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ function quit {
}
trap quit EXIT

# update links to point to the iptables binaries
iptables -V

# Start ovsdb
/usr/share/openvswitch/scripts/ovs-ctl restart --no-ovs-vswitchd --system-id=random
# Restrict the number of pthreads ovs-vswitchd creates to reduce the
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ require (
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08
github.com/containernetworking/cni v1.1.2
github.com/containernetworking/plugins v1.1.1
github.com/coreos/go-iptables v0.6.0
github.com/docker/docker v20.10.22+incompatible
github.com/emicklei/go-restful/v3 v3.10.1
github.com/evanphx/json-patch/v5 v5.6.0
github.com/greenpau/ovsdb v1.0.3
github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca
github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230327064018-0b27f88874f7
github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875
github.com/moby/sys/mountinfo v0.6.2
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -365,8 +365,6 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
Expand Down Expand Up @@ -896,6 +894,8 @@ github.com/kubeovn/arp v0.0.0-20230101053045-8a0772d9c34c h1:AcOKlV+lInNlGO3o3+1
github.com/kubeovn/arp v0.0.0-20230101053045-8a0772d9c34c/go.mod h1:Ce8lvkopTGXfPmeb5AY3/umEOmoFVV3HlCPGfGk0+Y0=
github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139 h1:MaLC8/dohKHU8nkfglfE2oikefB6urJG75yZDOcKTRU=
github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139/go.mod h1:ulxnUH9cbIOtCH+exhJPeV2mleh+bDv67WKsl/MVU/g=
github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca h1:fTMjoho2et9nKVOFrjzVEWVd9XD1zzxOYrlxxZpO0fU=
github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca/go.mod h1:jY1XeGzkx8ASNJ+SqQSxTESNXARkjvt+I6IJOTnzIjw=
github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230327064018-0b27f88874f7 h1:X5/DAYXXe8p3mUz3Z+j0dsgpIUPiNhaq0f7D1Z9/8CY=
github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230327064018-0b27f88874f7/go.mod h1:rNwHas8aX9k/BEz5dwObhRvfV7KEd0MnrTTDd4gQ3D0=
github.com/kubeovn/kubevirt-client-go v0.0.0-20221209084839-9c2ed1f0604d h1:sM7V2MhONBa10zYQA1yg/UbPm/Y7JqVqymtgoDiGqMo=
Expand Down
70 changes: 59 additions & 11 deletions pkg/daemon/controller_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"syscall"

"github.com/alauda/felix/ipsets"
"github.com/coreos/go-iptables/iptables"
"github.com/kubeovn/go-iptables/iptables"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
Expand All @@ -30,29 +30,77 @@ import (

// ControllerRuntime represents runtime specific controller members
type ControllerRuntime struct {
iptables map[string]*iptables.IPTables
ipsets map[string]*ipsets.IPSets
iptables map[string]*iptables.IPTables
iptablesObsolete map[string]*iptables.IPTables
ipsets map[string]*ipsets.IPSets
}

func evalCommandSymlinks(cmd string) (string, error) {
path, err := exec.LookPath(cmd)
if err != nil {
return "", fmt.Errorf("failed to search for command %q: %v", cmd, err)
}
file, err := filepath.EvalSymlinks(path)
if err != nil {
return "", fmt.Errorf("failed to read evaluate symbolic links for file %q: %v", path, err)
}

return file, nil
}

func isLegacyIptablesMode() (bool, error) {
path, err := evalCommandSymlinks("iptables")
if err != nil {
return false, err
}
pathLegacy, err := evalCommandSymlinks("iptables-legacy")
if err != nil {
return false, err
}
return path == pathLegacy, nil
}

func (c *Controller) initRuntime() error {
c.ControllerRuntime.iptables = make(map[string]*iptables.IPTables)
c.ControllerRuntime.ipsets = make(map[string]*ipsets.IPSets)
ok, err := isLegacyIptablesMode()
if err != nil {
klog.Errorf("failed to check iptables mode: %v", err)
return err
}
if !ok {
// iptables works in nft mode, we should migrate iptables rules
c.iptablesObsolete = make(map[string]*iptables.IPTables, 2)
}

c.iptables = make(map[string]*iptables.IPTables)
c.ipsets = make(map[string]*ipsets.IPSets)

if c.protocol == kubeovnv1.ProtocolIPv4 || c.protocol == kubeovnv1.ProtocolDual {
iptables, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
if err != nil {
return err
}
c.ControllerRuntime.iptables[kubeovnv1.ProtocolIPv4] = iptables
c.ControllerRuntime.ipsets[kubeovnv1.ProtocolIPv4] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV4, IPSetPrefix, nil, nil))
c.iptables[kubeovnv1.ProtocolIPv4] = ipt
if c.iptablesObsolete != nil {
if ipt, err = iptables.NewWithProtocolAndMode(iptables.ProtocolIPv4, "legacy"); err != nil {
return err
}
c.iptablesObsolete[kubeovnv1.ProtocolIPv4] = ipt
}
c.ipsets[kubeovnv1.ProtocolIPv4] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV4, IPSetPrefix, nil, nil))
}
if c.protocol == kubeovnv1.ProtocolIPv6 || c.protocol == kubeovnv1.ProtocolDual {
iptables, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
if err != nil {
return err
}
c.ControllerRuntime.iptables[kubeovnv1.ProtocolIPv6] = iptables
c.ControllerRuntime.ipsets[kubeovnv1.ProtocolIPv6] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV6, IPSetPrefix, nil, nil))
c.iptables[kubeovnv1.ProtocolIPv6] = ipt
if c.iptablesObsolete != nil {
if ipt, err = iptables.NewWithProtocolAndMode(iptables.ProtocolIPv6, "legacy"); err != nil {
return err
}
c.iptablesObsolete[kubeovnv1.ProtocolIPv6] = ipt
}
c.ipsets[kubeovnv1.ProtocolIPv6] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV6, IPSetPrefix, nil, nil))
}

return nil
Expand Down
Loading

0 comments on commit 3688ab3

Please sign in to comment.