Skip to content

Commit

Permalink
iptables: use the same mode with kube-proxy (#2535)
Browse files Browse the repository at this point in the history
  • Loading branch information
zhangzujian authored Apr 1, 2023
1 parent 0947798 commit 6d50bdc
Show file tree
Hide file tree
Showing 13 changed files with 537 additions and 223 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build-x86-image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1651,7 +1651,7 @@ jobs:
name: Kube-OVN Submariner Conformance E2E
needs:
- build-kube-ovn
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
- uses: actions/checkout@v3
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/scheduled-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1222,7 +1222,7 @@ jobs:

kube-ovn-submariner-conformance-e2e:
name: Kube-OVN Submariner Conformance E2E
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
timeout-minutes: 30
strategy:
fail-fast: false
Expand Down
1 change: 1 addition & 0 deletions dist/images/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ COPY grace_stop_ovn_controller /usr/share/ovn/scripts/grace_stop_ovn_controller

WORKDIR /kube-ovn

RUN /kube-ovn/iptables-wrapper-installer.sh --no-sanity-check
RUN rm -f /usr/bin/nc &&\
rm -f /usr/bin/netcat

Expand Down
3 changes: 0 additions & 3 deletions dist/images/Dockerfile.base
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,6 @@ RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname
tcpdump ipset curl uuid-runtime openssl inetutils-ping arping ndisc6 \
logrotate dnsutils net-tools -y --no-install-recommends && \
rm -rf /var/lib/apt/lists/* && \
cd /usr/sbin && \
ln -sf /usr/sbin/iptables-legacy iptables && \
ln -sf /usr/sbin/ip6tables-legacy ip6tables && \
rm -rf /etc/localtime

RUN mkdir -p /var/run/openvswitch && \
Expand Down
211 changes: 211 additions & 0 deletions dist/images/iptables-wrapper-installer.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
#!/bin/sh

# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Usage:
#
# iptables-wrapper-installer.sh [--no-sanity-check]
#
# Installs a wrapper iptables script in a container that will figure out
# whether iptables-legacy or iptables-nft is in use on the host and then
# replaces itself with the correct underlying iptables version.
#
# Unless "--no-sanity-check" is passed, it will first verify that the
# container already contains a suitable version of iptables.

# NOTE: This can only use POSIX /bin/sh features; the build container
# might not contain bash.

# original source:
# https://github.com/kubernetes-sigs/iptables-wrappers/blob/master/iptables-wrapper-installer.sh

set -eu

# Find iptables binary location
if [ -d /usr/sbin -a -e /usr/sbin/iptables ]; then
sbin="/usr/sbin"
elif [ -d /sbin -a -e /sbin/iptables ]; then
sbin="/sbin"
else
echo "ERROR: iptables is not present in either /usr/sbin or /sbin" 1>&2
exit 1
fi

# Determine how the system selects between iptables-legacy and iptables-nft
if [ -x /usr/sbin/alternatives ]; then
# Fedora/SUSE style alternatives
altstyle="fedora"
elif [ -x /usr/sbin/update-alternatives ]; then
# Debian style alternatives
altstyle="debian"
else
# No alternatives system
altstyle="none"
fi

if [ "${1:-}" != "--no-sanity-check" ]; then
# Ensure dependencies are installed
if ! version=$("${sbin}/iptables-nft" --version 2> /dev/null); then
echo "ERROR: iptables-nft is not installed" 1>&2
exit 1
fi
if ! "${sbin}/iptables-legacy" --version > /dev/null 2>&1; then
echo "ERROR: iptables-legacy is not installed" 1>&2
exit 1
fi

case "${version}" in
*v1.8.[0123]\ *)
echo "ERROR: iptables 1.8.0 - 1.8.3 have compatibility bugs." 1>&2
echo " Upgrade to 1.8.4 or newer." 1>&2
exit 1
;;
*)
# 1.8.4+ are OK
;;
esac
fi

# Start creating the wrapper...
rm -f "${sbin}/iptables-wrapper"
cat > "${sbin}/iptables-wrapper" <<EOF
#!/bin/sh
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# NOTE: This can only use POSIX /bin/sh features; the container image
# might not contain bash.
set -eu
# In kubernetes 1.17 and later, kubelet will have created at least
# one chain in the "mangle" table (either "KUBE-IPTABLES-HINT" or
# "KUBE-KUBELET-CANARY"), so check that first, against
# iptables-nft, because we can check that more efficiently and
# it's more common these days.
nft_kubelet_rules=\$( (iptables-nft-save -t mangle || true; ip6tables-nft-save -t mangle || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
if [ "\${nft_kubelet_rules}" -ne 0 ]; then
mode=nft
else
# Check for kubernetes 1.17-or-later with iptables-legacy. We
# can't pass "-t mangle" to iptables-legacy-save because it would
# cause the kernel to create that table if it didn't already
# exist, which we don't want. So we have to grab all the rules
legacy_kubelet_rules=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
if [ "\${legacy_kubelet_rules}" -ne 0 ]; then
mode=legacy
else
# With older kubernetes releases there may not be any _specific_
# rules we can look for, but we assume that some non-containerized process
# (possibly kubelet) will have created _some_ iptables rules.
num_legacy_lines=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l)
num_nft_lines=\$( (iptables-nft-save || true; ip6tables-nft-save || true) 2>/dev/null | grep '^-' | wc -l)
if [ "\${num_legacy_lines}" -gt "\${num_nft_lines}" ]; then
mode=legacy
else
mode=nft
fi
fi
fi
EOF

# Write out the appropriate alternatives-selection commands
case "${altstyle}" in
fedora)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
EOF
;;

debian)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
update-alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
update-alternatives --set ip6tables "/usr/sbin/ip6tables-\${mode}" > /dev/null || failed=1
EOF
;;

*)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
rm -f "${sbin}/\${cmd}"
ln -s "${sbin}/xtables-\${mode}-multi" "${sbin}/\${cmd}"
done 2>/dev/null || failed=1
EOF
;;
esac

# Write out the post-alternatives-selection error checking and final wrap-up
cat >> "${sbin}/iptables-wrapper" <<EOF
if [ "\${failed:-0}" = 1 ]; then
echo "Unable to redirect iptables binaries. (Are you running in an unprivileged pod?)" 1>&2
# fake it, though this will probably also fail if they aren't root
exec "${sbin}/xtables-\${mode}-multi" "\$0" "\$@"
fi
# Now re-exec the original command with the newly-selected alternative
exec "\$0" "\$@"
EOF
chmod +x "${sbin}/iptables-wrapper"

# Now back in the installer script, point the iptables binaries at our
# wrapper
case "${altstyle}" in
fedora)
alternatives \
--install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables iptables /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-save iptables-save /usr/sbin/iptables-wrapper
;;

debian)
update-alternatives \
--install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper
update-alternatives \
--install /usr/sbin/ip6tables ip6tables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/ip6tables-restore ip6tables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-save ip6tables-save /usr/sbin/iptables-wrapper
;;

*)
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
rm -f "${sbin}/${cmd}"
ln -s "${sbin}/iptables-wrapper" "${sbin}/${cmd}"
done
;;
esac

# Cleanup
rm -f "$0"
3 changes: 3 additions & 0 deletions dist/images/start-cniserver.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ do
fi
done

# update links to point to the iptables binaries
iptables -V

# If nftables not exist do not exit
set +e
iptables -P FORWARD ACCEPT
Expand Down
3 changes: 3 additions & 0 deletions dist/images/start-ovs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ function quit {
}
trap quit EXIT

# update links to point to the iptables binaries
iptables -V

# Start ovsdb
/usr/share/openvswitch/scripts/ovs-ctl restart --no-ovs-vswitchd --system-id=random
# Restrict the number of pthreads ovs-vswitchd creates to reduce the
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ require (
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08
github.com/containernetworking/cni v1.1.2
github.com/containernetworking/plugins v1.2.0
github.com/coreos/go-iptables v0.6.0
github.com/docker/docker v23.0.2+incompatible
github.com/emicklei/go-restful/v3 v3.10.2
github.com/evanphx/json-patch/v5 v5.6.0
github.com/go-logr/stdr v1.2.2
github.com/golang/mock v1.6.0
github.com/greenpau/ovsdb v1.0.3
github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca
github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230327064018-0b27f88874f7
github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875
github.com/moby/sys/mountinfo v0.6.2
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -378,8 +378,6 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
Expand Down Expand Up @@ -924,6 +922,8 @@ github.com/kubeovn/arp v0.0.0-20230101053045-8a0772d9c34c h1:AcOKlV+lInNlGO3o3+1
github.com/kubeovn/arp v0.0.0-20230101053045-8a0772d9c34c/go.mod h1:Ce8lvkopTGXfPmeb5AY3/umEOmoFVV3HlCPGfGk0+Y0=
github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139 h1:MaLC8/dohKHU8nkfglfE2oikefB6urJG75yZDOcKTRU=
github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139/go.mod h1:ulxnUH9cbIOtCH+exhJPeV2mleh+bDv67WKsl/MVU/g=
github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca h1:fTMjoho2et9nKVOFrjzVEWVd9XD1zzxOYrlxxZpO0fU=
github.com/kubeovn/go-iptables v0.0.0-20230322103850-8619a8ab3dca/go.mod h1:jY1XeGzkx8ASNJ+SqQSxTESNXARkjvt+I6IJOTnzIjw=
github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230327064018-0b27f88874f7 h1:X5/DAYXXe8p3mUz3Z+j0dsgpIUPiNhaq0f7D1Z9/8CY=
github.com/kubeovn/gonetworkmanager/v2 v2.0.0-20230327064018-0b27f88874f7/go.mod h1:rNwHas8aX9k/BEz5dwObhRvfV7KEd0MnrTTDd4gQ3D0=
github.com/kubeovn/kubevirt-client-go v0.0.0-20230301060230-5aae21a9918b h1:lTSeHcqlNKqoeqsaDLGb2iyFcIC0EaT+AtXD+Rzdpi0=
Expand Down
Loading

0 comments on commit 6d50bdc

Please sign in to comment.