From 2430189d2645d221001bd37f35e9ab4e03faa4a4 Mon Sep 17 00:00:00 2001 From: Naman Agarwal Date: Mon, 23 Oct 2023 14:40:21 +0530 Subject: [PATCH] Enable Windows OVS Container to run on pristine Host Environment Support installation of signed ovs kernel driver from inside container when no installed on host. Signed-off-by: Naman Agarwal --- .../antrea-windows-containerd-with-ovs.yml | 19 ++++++++++-- .../conf/Run-AntreaOVS-Containerd.ps1 | 17 +++++++++- ci/jenkins/test.sh | 23 +++++++++----- docs/windows.md | 31 +++++++++++++------ hack/windows/Prepare-AntreaAgent.ps1 | 13 +++++--- 5 files changed, 78 insertions(+), 25 deletions(-) diff --git a/build/yamls/antrea-windows-containerd-with-ovs.yml b/build/yamls/antrea-windows-containerd-with-ovs.yml index 8b08878b27d..fd22e58967d 100644 --- a/build/yamls/antrea-windows-containerd-with-ovs.yml +++ b/build/yamls/antrea-windows-containerd-with-ovs.yml @@ -25,16 +25,31 @@ data: $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT $mountPath = ($mountPath.Replace('\', '/')).TrimEnd('/') $env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/openvswitch/usr/bin;$mountPath/openvswitch/usr/sbin" + $OVSDriverDir="$mountPath\openvswitch\driver" + + # Check if OVSExt driver is already installed + $driverStatus = netcfg -q ovsext + if ($driverStatus -like '*not installed*') { + # Install OVS Driver + netcfg -l $OVSDriverDir/ovsext.inf -c s -i OVSExt + } + + # Configure OVS processes $OVS_DB_SCHEMA_PATH = "$mountPath/openvswitch/usr/share/openvswitch/vswitch.ovsschema" $OVS_DB_PATH = "C:\openvswitch\etc\openvswitch\conf.db" if ($(Test-Path $OVS_DB_SCHEMA_PATH) -and !$(Test-Path $OVS_DB_PATH)) { + mkdir -force C:\openvswitch\etc\openvswitch\ ovsdb-tool create "$OVS_DB_PATH" "$OVS_DB_SCHEMA_PATH" } + $OVS_RUN_PATH = "C:\openvswitch\var\run\openvswitch" + if (!$(Test-Path $OVS_RUN_PATH)) { + mkdir -force $OVS_RUN_PATH + } ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach ovs-vsctl --no-wait init # Set OVS version. - $OVS_VERSION=$(Get-Item $mountPath\openvswitch\driver\OVSExt.sys).VersionInfo.ProductVersion + $OVS_VERSION=$(Get-Item $OVSDriverDir\OVSExt.sys).VersionInfo.ProductVersion ovs-vsctl --no-wait set Open_vSwitch . ovs_version=$OVS_VERSION ovs-vswitchd --log-file=/var/log/antrea/openvswitch/ovs-vswitchd.log --pidfile -vfile:info --detach @@ -255,7 +270,7 @@ spec: metadata: annotations: checksum/agent-windows: 5afdcec25b0f1e6a241dff1bc0c7484cf5044eb33086b2a84961610fec15b461 - checksum/windows-config: 6be75a1fcb4855e2e138e36109862d93f16e2d654312c74532bbcc64c65af3d9 + checksum/windows-config: 6bd9e27a05233439f0012137ccfa6dc3817f99df9e394c9d07922c3277e433d1 microsoft.com/hostprocess-inherit-user: "true" labels: app: antrea diff --git a/build/yamls/windows/containerd-with-ovs/conf/Run-AntreaOVS-Containerd.ps1 b/build/yamls/windows/containerd-with-ovs/conf/Run-AntreaOVS-Containerd.ps1 index 4f8012b58f1..1d61a628645 100644 --- a/build/yamls/windows/containerd-with-ovs/conf/Run-AntreaOVS-Containerd.ps1 +++ b/build/yamls/windows/containerd-with-ovs/conf/Run-AntreaOVS-Containerd.ps1 @@ -2,16 +2,31 @@ $ErrorActionPreference = "Stop" $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT $mountPath = ($mountPath.Replace('\', '/')).TrimEnd('/') $env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/openvswitch/usr/bin;$mountPath/openvswitch/usr/sbin" +$OVSDriverDir="$mountPath\openvswitch\driver" + +# Check if OVSExt driver is already installed +$driverStatus = netcfg -q ovsext +if ($driverStatus -like '*not installed*') { + # Install OVS Driver + netcfg -l $mountPath/openvswitch/driver/ovsext.inf -c s -i OVSExt +} + +#Configure OVS processes $OVS_DB_SCHEMA_PATH = "$mountPath/openvswitch/usr/share/openvswitch/vswitch.ovsschema" $OVS_DB_PATH = "C:\openvswitch\etc\openvswitch\conf.db" if ($(Test-Path $OVS_DB_SCHEMA_PATH) -and !$(Test-Path $OVS_DB_PATH)) { + mkdir -force C:\openvswitch\etc\openvswitch\ ovsdb-tool create "$OVS_DB_PATH" "$OVS_DB_SCHEMA_PATH" } +$OVS_RUN_PATH = "C:\openvswitch\var\run\openvswitch" +if (!$(Test-Path $OVS_RUN_PATH)) { + mkdir -force $OVS_RUN_PATH +} ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach ovs-vsctl --no-wait init # Set OVS version. -$OVS_VERSION=$(Get-Item $mountPath\openvswitch\driver\OVSExt.sys).VersionInfo.ProductVersion +$OVS_VERSION=$(Get-Item $OVSDriverDir\OVSExt.sys).VersionInfo.ProductVersion ovs-vsctl --no-wait set Open_vSwitch . ovs_version=$OVS_VERSION ovs-vswitchd --log-file=/var/log/antrea/openvswitch/ovs-vswitchd.log --pidfile -vfile:info --detach diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index 2cc7341bccd..e112598f816 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -152,7 +152,12 @@ export NO_PULL E2ETEST_PATH=${WORKDIR}/kubernetes/_output/dockerized/bin/linux/amd64/e2e.test function export_govc_env_var { - export GOVC_URL=$GOVC_URL + env_govc="${WORKDIR}/govc.env" + if [ -f "$env_govc" ]; then + source "$env_govc" + else + export GOVC_URL=$GOVC_URL + fi export GOVC_USERNAME=$GOVC_USERNAME export GOVC_PASSWORD=$GOVC_PASSWORD export GOVC_INSECURE=1 @@ -322,7 +327,11 @@ function prepare_env { function revert_snapshot_windows { WIN_NAME=$1 echo "==== Reverting Windows VM ${WIN_NAME} =====" - govc snapshot.revert -vm ${WIN_NAME} win-initial + if [[ $WIN_NAME == *"jumper"* ]]; then + govc snapshot.revert -vm ${WIN_NAME} win-initial + else + govc snapshot.revert -vm ${WIN_NAME} pristine-win-initial + fi # If Windows VM fails to power on correctly in time, retry several times. winVMIPs="" for i in `seq 10`; do @@ -516,14 +525,12 @@ function build_and_deliver_antrea_windows_and_linux_docker_images { function build_and_deliver_antrea_windows_and_linux_containerd_images { echo "====== Cleanup Antrea Installation Before Delivering Antrea Windows and Antrea Linux containerd Images =====" clean_antrea - kubectl delete -f ${WORKDIR}/antrea-windows-containerd.yml --ignore-not-found=true || true + kubectl delete -f ${WORKDIR}/antrea-windows-containerd-with-ovs.yml --ignore-not-found=true || true kubectl delete -f ${WORKDIR}/kube-proxy-windows-containerd.yml --ignore-not-found=true || true kubectl delete daemonset antrea-agent -n kube-system --ignore-not-found=true || true kubectl delete -f ${WORKDIR}/antrea.yml --ignore-not-found=true || true prepare_env - ${CLEAN_STALE_IMAGES_CONTAINERD} - ${PRINT_CONTAINERD_STATUS} chmod -R g-w build/images/ovs chmod -R g-w build/images/base # Clean docker image to save disk space. @@ -531,11 +538,11 @@ function build_and_deliver_antrea_windows_and_linux_containerd_images { ${PRINT_DOCKER_STATUS} export_govc_env_var # Enable verbose log for troubleshooting. - sed -i "s/--v=0/--v=4/g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd.yml + sed -i "s/--v=0/--v=4/g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd-with-ovs.yml echo "====== Updating yaml files to enable proxyAll ======" KUBE_API_SERVER=$(kubectl --kubeconfig=$KubeConfigFile config view -o jsonpath='{.clusters[0].cluster.server}') - sed -i "s|.*kubeAPIServerOverride: \"\"| kubeAPIServerOverride: \"${KUBE_API_SERVER}\"|g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd.yml + sed -i "s|.*kubeAPIServerOverride: \"\"| kubeAPIServerOverride: \"${KUBE_API_SERVER}\"|g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd-with-ovs.yml cp -f build/yamls/*.yml $WORKDIR set +e @@ -1148,7 +1155,7 @@ fi trap clean_antrea EXIT if [[ ${TESTCASE} =~ "windows" ]]; then if [[ ${TESTCASE} =~ "containerd" ]]; then - WINDOWS_YAML_SUFFIX="windows-containerd" + WINDOWS_YAML_SUFFIX="windows-containerd-with-ovs" build_and_deliver_antrea_windows_and_linux_containerd_images if [[ ${TESTCASE} =~ "e2e" ]]; then run_e2e_windows diff --git a/docs/windows.md b/docs/windows.md index 7a3763c7d26..88a6b445309 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -222,12 +222,25 @@ kubectl apply -f - ##### 1. (Optional) Install OVS (provided by Antrea or your own) +Antrea supports running OVS on Windows as native services or inside a +host-process container. If you have an OVS package with a signed kernel +driver and want to run OVS inside container, you can skip this step. + Antrea provides a pre-built OVS package which contains test-signed OVS kernel driver. If you don't have a self-signed OVS package and just want to try the Antrea on Windows, this package can be used for testing. We also provide a helper script `Install-OVS.ps1` to install the OVS driver and register userspace binaries -as services. If you want to use your own signed OVS package for production, you can -run `Install-OVS.ps1` like this: +as services. + +If you want to containerize OVS with an unsigned kernel driver, you must +pre-install the driver on the worker node before joining cluster. Hence, +you need to run the `Install-OVS.ps1` script to install only the driver like this: + +```powershell +.\Install-OVS.ps1 -InstallUserspace $false +``` + +If you want to run OVS as Windows native services, you can run the script like this, ```powershell Install-OVS.ps1 -ImportCertificate $false -Local -LocalFile @@ -256,13 +269,6 @@ get-service ovsdb-server get-service ovs-vswitchd ``` -If you want to containerize OVS for containerd runtime, OVS userspace processes are -not run on the host and hence you can set the `InstallUserspace` parameter to false. - -```powershell -.\Install-OVS.ps1 -InstallUserspace $false -``` - ##### 2. Disable Windows Firewall ```powershell @@ -340,6 +346,13 @@ The script `Prepare-AntreaAgent.ps1` performs following tasks: After the Windows Node reboots, there will be stale network resources which need to be cleaned before starting antrea-agent. +* Ensure OVS services are running. + + This script starts OVS services on the Node if they are not running. This step + needs to be skipped in case of OVS containerization. Hence, you need to specify + the parameter ContainerizeOVS as true by executing the command + `.\Prepare-AntreaAgent.ps1 -ContainerizeOVS $true`. + As you know from the task details from above, the script must be executed every time you restart the Node to prepare the environment for antrea-agent. diff --git a/hack/windows/Prepare-AntreaAgent.ps1 b/hack/windows/Prepare-AntreaAgent.ps1 index a5aff40d126..ced3dc5e4ac 100644 --- a/hack/windows/Prepare-AntreaAgent.ps1 +++ b/hack/windows/Prepare-AntreaAgent.ps1 @@ -13,7 +13,8 @@ Specifies whether kube-proxy interface is included in the installation. If false be installed on the host. #> Param( - [parameter(Mandatory = $false)] [bool] $InstallKubeProxy = $true + [parameter(Mandatory = $false)] [bool] $InstallKubeProxy = $true, + [parameter(Mandatory = $false)] [bool] $ContainerizeOVS = $false ) $ErrorActionPreference = 'Stop' @@ -35,10 +36,12 @@ if ($NeedCleanNetwork) { & $CleanAntreaNetworkScript } # Enure OVS services are running. -Write-Host "Starting ovsdb-server service..." -Start-Service ovsdb-server -Write-Host "Starting ovs-vswitchd service..." -Start-Service ovs-vswitchd +if ($ContainerizeOVS -eq $false) { + Write-Host "Starting ovsdb-server service..." + Start-Service ovsdb-server + Write-Host "Starting ovs-vswitchd service..." + Start-Service ovs-vswitchd +} # Prepare service network interface for kube-proxy. if ($InstallKubeProxy -eq $true) { Write-Host "Preparing service network interface for kube-proxy..."