diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 71c86d2657b..f33f0640259 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -160,3 +160,22 @@ jobs: run: | echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin docker push antrea/flow-aggregator:latest + + build-antrea-migrator: + needs: check-changes + if: ${{ needs.check-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }} + runs-on: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + with: + show-progress: false + - name: Build antrea-migrator Docker image + run: make build-migrator + - name: Push antrea-migrator Docker image to registry + if: ${{ github.repository == 'antrea-io/antrea' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + docker push antrea/antrea-migrator:latest diff --git a/Makefile b/Makefile index e612daeb145..0bed0777b0d 100644 --- a/Makefile +++ b/Makefile @@ -380,6 +380,16 @@ build-scale-simulator: docker build -t antrea/antrea-ubuntu-simulator:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.simulator.build.ubuntu $(DOCKER_BUILD_ARGS) . docker tag antrea/antrea-ubuntu-simulator:$(DOCKER_IMG_VERSION) antrea/antrea-ubuntu-simulator +.PHONY: build-migrator +build-migrator: + @echo "===> Building antrea/antrea-migrator Docker image <===" +ifneq ($(NO_PULL),) + docker build -t antrea/antrea-migrator:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.build.migrator $(DOCKER_BUILD_ARGS) . +else + docker build --pull -t antrea/antrea-migrator:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.build.migrator $(DOCKER_BUILD_ARGS) . +endif + docker tag antrea/antrea-migrator:$(DOCKER_IMG_VERSION) antrea/antrea-migrator + .PHONY: manifest manifest: @echo "===> Generating dev manifest for Antrea <===" diff --git a/build/images/Dockerfile.build.migrator b/build/images/Dockerfile.build.migrator new file mode 100644 index 00000000000..d2fa0f8d079 --- /dev/null +++ b/build/images/Dockerfile.build.migrator @@ -0,0 +1,33 @@ +# Copyright 2023 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM antrea/toolbox:1.2 + +LABEL maintainer="Antrea " +LABEL description="The Docker image to migrate other CNIs to Antrea CNI." + +USER root + +COPY build/images/scripts/migrate_cni /usr/local/bin/ + +ENV CRICTL_VERSION="v1.27.0" +RUN apt update \ + && apt install -y jq ca-certificates \ + && apt clean \ + && wget -q https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz \ + && tar -xzf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz \ + && mv crictl /usr/local/bin/ \ + && rm -f crictl-${CRICTL_VERSION}-linux-amd64.tar.gz \ + && chmod +x /usr/local/bin/crictl \ + && chmod +x /usr/local/bin/migrate_cni diff --git a/build/images/scripts/migrate_cni b/build/images/scripts/migrate_cni new file mode 100755 index 00000000000..416bbd6af65 --- /dev/null +++ b/build/images/scripts/migrate_cni @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Copyright 2023 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CRICTL="crictl --runtime-endpoint unix:///run/containerd/containerd.sock" + +for file in /etc/cni/net.d/*; do + if [ "$file" != "/etc/cni/net.d/10-antrea.conflist" ]; then + mv "$file" "$file".bak + fi +done + +# Remove rules added by the previous CNI in CHAIN CNI-HOSTPORT-DNAT. +# CHAIN CNI-HOSTPORT-DNAT is created by portmap for ingress controllers. It will not be deleted +# even if the CNI is removed, so we delete the stale rules here. +if iptables -t nat -S | grep -q "CNI-HOSTPORT-DNAT"; then + rules=$(iptables -t nat -L CNI-HOSTPORT-DNAT --line-numbers | grep -v "antrea" | awk 'NR>2 {print $1}') + chains=$(iptables -t nat -L CNI-HOSTPORT-DNAT | grep -v "antrea" | awk 'NR>2 {print $1}') + for rule in $rules; do + iptables -t nat -D CNI-HOSTPORT-DNAT "$rule" + done + for chain in $chains; do + iptables -t nat -F "$chain" + iptables -t nat -X "$chain" + done +fi + +pods=$($CRICTL pods -q) +for pod in $pods; do + network_type=$($CRICTL inspectp $pod | jq -r .status.linux.namespaces.options.network) + if [ "$network_type" == "POD" ]; then + $CRICTL stopp $pod && $CRICTL rmp $pod + fi +done diff --git a/build/yamls/antrea-migrator.yml b/build/yamls/antrea-migrator.yml new file mode 100644 index 00000000000..7ed52c67c24 --- /dev/null +++ b/build/yamls/antrea-migrator.yml @@ -0,0 +1,103 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: antrea-migrator + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: antrea-migrator + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - create + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create + - apiGroups: + - "apps/v1" + resources: + - daemonsets + verbs: + - create +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: antrea-migrator + namespace: kube-system +subjects: + - kind: ServiceAccount + name: antrea-migrator + namespace: kube-system +roleRef: + kind: ClusterRole + name: antrea-migrator + apiGroup: rbac.authorization.k8s.io +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + labels: + app: antrea + component: antrea-migrator + name: antrea-migrator + namespace: kube-system +spec: + selector: + matchLabels: + app: antrea + component: antrea-migrator + template: + metadata: + labels: + app: antrea + component: antrea-migrator + spec: + hostPID: true + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: antrea-migrator + volumes: + - name: cni-cfg + hostPath: + path: /etc/cni/net.d + - name: containerd + hostPath: + path: /run/containerd + initContainers: + - name: antrea-migrator-init + image: antrea/antrea-migrator:latest + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - migrate_cni + volumeMounts: + - mountPath: /run/containerd + name: containerd + - mountPath: /etc/cni/net.d + name: cni-cfg + containers: + - image: antrea/antrea-migrator:latest + imagePullPolicy: IfNotPresent + name: antrea-migrator + command: + - "sleep" + - "infinity" diff --git a/docs/migrate-to-antrea.md b/docs/migrate-to-antrea.md new file mode 100644 index 00000000000..e7f12114a36 --- /dev/null +++ b/docs/migrate-to-antrea.md @@ -0,0 +1,73 @@ +# Migrate from another CNI to Antrea + +This document describes how to migrate from other CNIs to Antrea. + +NOTE: The following is a reference list of CNIs and versions we have already +verified the migration process. CNIs and versions that are not listed here +might also work. Please create an issue if you run into problems during the +migration to Antrea. During the migration process, no Kubernetes resources +should be created or deleted, otherwise the migration process might fail or +some unexpected problems might occur. + +| CNI | Version | +|---------|---------| +| Calico | v3.26 | +| Flannel | v0.22.0 | + +The migration process is divided into three steps: + +1. Clean up old CNI. +2. Install Antrea in the cluster. +3. Restart all Pods in the cluster in-place. + +## Clean up the old CNI + +Before installing Antrea in the cluster, the old CNI needs to be cleaned up. The +cleanup process is different for different CNIs, but the basic idea is to +remove the old CNI DaemonSet, Deployment, and CRDs from the cluster. The most +common way is to use the `kubectl delete` command to delete the old CNI, just +like the following example: + +```bash +$ kubectl delete -f +``` + +## Install Antrea + +The second step is to install Antrea in the cluster. You can follow the +[installation guide](https://github.com/antrea-io/antrea/blob/main/docs/getting-started.md) +to install Antrea. The following is an example of installing Antrea v1.14.1: + +```bash +$ kubectl apply -f https://github.com/antrea-io/antrea/releases/download/v1.14.1/antrea.yml +``` + +## Restart Pods + +After Antrea is installed in the cluster, the next step is to restart all +Pods in the cluster in-place by the following command. This step will create +a DaemonSet in the cluster, which will restart all Pods in the cluster in-place. + +```bash +$ kubectl apply -f https://github.com/antrea-io/antrea/releases/download/v1.15.0/antrea-migrator.yml +``` + +The reason for restarting all Pods is that Antrea needs to take over the +network management and IPAM from the old CNI. In order to avoid the Pods +being rescheduled, we restart all Pods in-place by deploying a DaemonSet +named `antrea-migrator`, which will run a Pod on each Node. The +`antrea-migrator` Pod will stop all containerd tasks of Pods with CNI network on +each Node, and the containerd tasks will be restarted by the containerd +service. In this way, all Pods in the cluster will not be rescheduled but will +be restarted in-place. + +The restart result will be similar as follows, and every Pod's `RESTARTS` will +be increased by 1: + +```bash +$ kubectl get pod -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +migrate-example-6d6b97f96b-29qbq 1/1 Running 1 (24s ago) 2m5s 10.10.1.3 test-worker +migrate-example-6d6b97f96b-dqx2g 1/1 Running 1 (23s ago) 2m5s 10.10.1.6 test-worker +migrate-example-6d6b97f96b-jpflg 1/1 Running 1 (23s ago) 2m5s 10.10.1.5 test-worker +``` diff --git a/hack/release/prepare-assets.sh b/hack/release/prepare-assets.sh index bcd537ec356..41aeab91117 100755 --- a/hack/release/prepare-assets.sh +++ b/hack/release/prepare-assets.sh @@ -107,6 +107,8 @@ export IMG_TAG=$VERSION export IMG_NAME=projects.registry.vmware.com/antrea/antrea-ubuntu ./hack/generate-standard-manifests.sh --mode release --out "$OUTPUT_DIR" +sed "s|antrea\/antrea-migrator:latest|$(echo $IMG_NAME | sed "s/ubuntu/migrator/"):$VERSION|g" ./build/yamls/antrea-migrator.yml > "$OUTPUT_DIR"/antrea-migrator.yml + export IMG_NAME=projects.registry.vmware.com/antrea/antrea-windows ./hack/generate-manifest-windows.sh --mode release > "$OUTPUT_DIR"/antrea-windows.yml ./hack/generate-manifest-windows.sh --mode release --containerd > "$OUTPUT_DIR"/antrea-windows-containerd.yml