Add script for migrating from other CNI to Antrea

Add a YAML file antrea-migrator.yml to migrate clusters
with other CNIs to Antrea. It will restart all Pods in-place.

A new image "antrea-migrator" is responsible for restarting
all Pods on each Nodes. It is a DaemonSet that tries to kill
sandboxes to restart the Pods in-place.

Signed-off-by: hjiajing <hjiajing@vmware.com>

.github/workflows/build.yml

@@ -160,3 +160,22 @@ jobs:
       run: |
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/flow-aggregator:latest
+
+  build-antrea-migrator:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
+    runs-on: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        show-progress: false
+    - name: Build antrea-migrator Docker image
+      run: make build-migrator
+    - name: Push antrea-migrator Docker image to registry
+      if: ${{ github.repository == 'antrea-io/antrea' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: |
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/antrea-migrator:latest

Makefile

@@ -380,6 +380,16 @@ build-scale-simulator:
 	docker build -t antrea/antrea-ubuntu-simulator:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.simulator.build.ubuntu $(DOCKER_BUILD_ARGS) .
 	docker tag antrea/antrea-ubuntu-simulator:$(DOCKER_IMG_VERSION) antrea/antrea-ubuntu-simulator
 
+.PHONY: build-migrator
+build-migrator:
+	@echo "===> Building antrea/antrea-migrator Docker image <==="
+ifneq ($(NO_PULL),)
+	docker build -t antrea/antrea-migrator:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.build.migrator $(DOCKER_BUILD_ARGS) .
+else
+	docker build --pull -t antrea/antrea-migrator:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.build.migrator $(DOCKER_BUILD_ARGS) .
+endif
+	docker tag antrea/antrea-migrator:$(DOCKER_IMG_VERSION) antrea/antrea-migrator
+
 .PHONY: manifest
 manifest:
 	@echo "===> Generating dev manifest for Antrea <==="

build/images/Dockerfile.build.migrator

@@ -0,0 +1,33 @@
+# Copyright 2023 Antrea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM antrea/toolbox:1.2
+
+LABEL maintainer="Antrea <projectantrea-dev@googlegroups.com>"
+LABEL description="The Docker image to migrate other CNIs to Antrea CNI."
+
+USER root
+
+COPY build/images/scripts/migrate_cni /usr/local/bin/
+
+ENV CRICTL_VERSION="v1.27.0"
+RUN apt update \
+    && apt install -y jq ca-certificates \
+    && apt clean \
+    && wget -q https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz \
+    && tar -xzf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz \
+    && mv crictl /usr/local/bin/ \
+    && rm -f crictl-${CRICTL_VERSION}-linux-amd64.tar.gz \
+    && chmod +x /usr/local/bin/crictl \
+    && chmod +x /usr/local/bin/migrate_cni

build/images/scripts/migrate_cni

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 Antrea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+CRICTL="crictl --runtime-endpoint unix:///run/containerd/containerd.sock"
+
+for file in /etc/cni/net.d/*; do
+    if [ "$file" != "/etc/cni/net.d/10-antrea.conflist" ]; then
+        mv "$file" "$file".bak
+    fi
+done
+
+# Remove rules added by the previous CNI in CHAIN CNI-HOSTPORT-DNAT.
+# CHAIN CNI-HOSTPORT-DNAT is created by portmap for ingress controllers. It will not be deleted
+# even if the CNI is removed, so we delete the stale rules here.
+if iptables -t nat -S | grep -q "CNI-HOSTPORT-DNAT"; then
+    rules=$(iptables -t nat -L CNI-HOSTPORT-DNAT --line-numbers | grep -v "antrea" | awk 'NR>2 {print $1}')
+    chains=$(iptables -t nat -L CNI-HOSTPORT-DNAT | grep -v "antrea" | awk 'NR>2 {print $1}')
+    for rule in $rules; do
+        iptables -t nat -D CNI-HOSTPORT-DNAT "$rule"
+    done
+    for chain in $chains; do
+        iptables -t nat -F "$chain"
+        iptables -t nat -X "$chain"
+    done
+fi
+
+pods=$($CRICTL pods -q)
+for pod in $pods; do
+    network_type=$($CRICTL inspectp $pod | jq -r .status.linux.namespaces.options.network)
+    if [ "$network_type" == "POD" ]; then
+        $CRICTL stopp $pod && $CRICTL rmp $pod
+    fi
+done

build/yamls/antrea-migrator.yml

@@ -0,0 +1,103 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: antrea-migrator
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: antrea-migrator
+  namespace: kube-system
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - create
+  - apiGroups:
+      - ""
+    resources:
+      - pods/exec
+    verbs:
+      - create
+  - apiGroups:
+      - "apps/v1"
+    resources:
+      - daemonsets
+    verbs:
+      - create
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: antrea-migrator
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: antrea-migrator
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: antrea-migrator
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  labels:
+    app: antrea
+    component: antrea-migrator
+  name: antrea-migrator
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: antrea
+      component: antrea-migrator
+  template:
+    metadata:
+      labels:
+        app: antrea
+        component: antrea-migrator
+    spec:
+      hostPID: true
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      serviceAccountName: antrea-migrator
+      volumes:
+        - name: cni-cfg
+          hostPath:
+            path: /etc/cni/net.d
+        - name: containerd
+          hostPath:
+            path: /run/containerd
+      initContainers:
+        - name: antrea-migrator-init
+          image: antrea/antrea-migrator:latest
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            privileged: true
+          command:
+            - migrate_cni
+          volumeMounts:
+            - mountPath: /run/containerd
+              name: containerd
+            - mountPath: /etc/cni/net.d
+              name: cni-cfg
+      containers:
+        - image: antrea/antrea-migrator:latest
+          imagePullPolicy: IfNotPresent
+          name: antrea-migrator
+          command:
+            - "sleep"
+            - "infinity"

docs/migrate-to-antrea.md

@@ -0,0 +1,73 @@
+# Migrate from another CNI to Antrea
+
+This document describes how to migrate from other CNIs to Antrea.
+
+NOTE: The following is a reference list of CNIs and versions we have already
+verified the migration process. CNIs and versions that are not listed here
+might also work. Please create an issue if you run into problems during the
+migration to Antrea. During the migration process, no Kubernetes resources
+should be created or deleted, otherwise the migration process might fail or
+some unexpected problems might occur.
+
+| CNI     | Version |
+|---------|---------|
+| Calico  | v3.26   |
+| Flannel | v0.22.0 |
+
+The migration process is divided into three steps:
+
+1. Clean up old CNI.
+2. Install Antrea in the cluster.
+3. Restart all Pods in the cluster in-place.
+
+## Clean up the old CNI
+
+Before installing Antrea in the cluster, the old CNI needs to be cleaned up. The
+cleanup process is different for different CNIs, but the basic idea is to
+remove the old CNI DaemonSet, Deployment, and CRDs from the cluster. The most
+common way is to use the `kubectl delete` command to delete the old CNI, just
+like the following example:
+
+```bash
+$ kubectl delete -f <OLD_CNI_MANIFEST>
+```
+
+## Install Antrea
+
+The second step is to install Antrea in the cluster. You can follow the
+[installation guide](https://github.com/antrea-io/antrea/blob/main/docs/getting-started.md)
+to install Antrea. The following is an example of installing Antrea v1.14.1:
+
+```bash
+$ kubectl apply -f https://github.com/antrea-io/antrea/releases/download/v1.14.1/antrea.yml
+```
+
+## Restart Pods
+
+After Antrea is installed in the cluster, the next step is to restart all
+Pods in the cluster in-place by the following command. This step will create
+a DaemonSet in the cluster, which will restart all Pods in the cluster in-place.
+
+```bash
+$ kubectl apply -f https://github.com/antrea-io/antrea/releases/download/v1.15.0/antrea-migrator.yml
+```
+
+The reason for restarting all Pods is that Antrea needs to take over the
+network management and IPAM from the old CNI. In order to avoid the Pods
+being rescheduled, we restart all Pods in-place by deploying a DaemonSet
+named `antrea-migrator`, which will run a Pod on each Node. The
+`antrea-migrator` Pod will stop all containerd tasks of Pods with CNI network on
+each Node, and the containerd tasks will be restarted by the containerd
+service. In this way, all Pods in the cluster will not be rescheduled but will
+be restarted in-place.
+
+The restart result will be similar as follows, and every Pod's `RESTARTS` will
+be increased by 1:
+
+```bash
+$ kubectl get pod -o wide
+NAME                               READY   STATUS    RESTARTS      AGE    IP          NODE          NOMINATED NODE   READINESS GATES
+migrate-example-6d6b97f96b-29qbq   1/1     Running   1 (24s ago)   2m5s   10.10.1.3   test-worker   <none>           <none>
+migrate-example-6d6b97f96b-dqx2g   1/1     Running   1 (23s ago)   2m5s   10.10.1.6   test-worker   <none>           <none>
+migrate-example-6d6b97f96b-jpflg   1/1     Running   1 (23s ago)   2m5s   10.10.1.5   test-worker   <none>           <none>
+```

hack/release/prepare-assets.sh

@@ -107,6 +107,8 @@ export IMG_TAG=$VERSION
 export IMG_NAME=projects.registry.vmware.com/antrea/antrea-ubuntu
 ./hack/generate-standard-manifests.sh --mode release --out "$OUTPUT_DIR"
 
+sed "s|antrea\/antrea-migrator:latest|$(echo $IMG_NAME | sed "s/ubuntu/migrator/"):$VERSION|g" ./build/yamls/antrea-migrator.yml > "$OUTPUT_DIR"/antrea-migrator.yml
+
 export IMG_NAME=projects.registry.vmware.com/antrea/antrea-windows
 ./hack/generate-manifest-windows.sh --mode release > "$OUTPUT_DIR"/antrea-windows.yml
 ./hack/generate-manifest-windows.sh --mode release --containerd > "$OUTPUT_DIR"/antrea-windows-containerd.yml