Enable Windows OVS Container to run on pristine Host Environment

Support installation of signed ovs kernel driver from inside container when no installed on host.

Signed-off-by: Naman Agarwal <naman.agarwal75@gmail.com>

build/yamls/antrea-windows-containerd-with-ovs.yml

@@ -25,16 +25,31 @@ data:
     $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
     $mountPath = ($mountPath.Replace('\', '/')).TrimEnd('/')
     $env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/openvswitch/usr/bin;$mountPath/openvswitch/usr/sbin"
+    $OVSDriverDir="$mountPath\openvswitch\driver"
+
+    # Check if OVSExt driver is already installed
+    $driverStatus = netcfg -q ovsext
+    if ($driverStatus -like '*not installed*') {
+      # Install OVS Driver
+      netcfg -l $OVSDriverDir/ovsext.inf -c s -i OVSExt
+    }
+
+    # Configure OVS processes
     $OVS_DB_SCHEMA_PATH = "$mountPath/openvswitch/usr/share/openvswitch/vswitch.ovsschema"
     $OVS_DB_PATH = "C:\openvswitch\etc\openvswitch\conf.db"
     if ($(Test-Path $OVS_DB_SCHEMA_PATH) -and !$(Test-Path $OVS_DB_PATH)) {
+        mkdir -force C:\openvswitch\etc\openvswitch\
         ovsdb-tool create "$OVS_DB_PATH" "$OVS_DB_SCHEMA_PATH"
     }
+    $OVS_RUN_PATH = "C:\openvswitch\var\run\openvswitch"
+    if (!$(Test-Path $OVS_RUN_PATH)) {
+      mkdir -force $OVS_RUN_PATH
+    }
     ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach
     ovs-vsctl --no-wait init
 
     # Set OVS version.
-    $OVS_VERSION=$(Get-Item $mountPath\openvswitch\driver\OVSExt.sys).VersionInfo.ProductVersion
+    $OVS_VERSION=$(Get-Item $OVSDriverDir\OVSExt.sys).VersionInfo.ProductVersion
     ovs-vsctl --no-wait set Open_vSwitch . ovs_version=$OVS_VERSION
 
     ovs-vswitchd --log-file=/var/log/antrea/openvswitch/ovs-vswitchd.log --pidfile -vfile:info --detach
@@ -258,7 +273,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/agent-windows: 5afdcec25b0f1e6a241dff1bc0c7484cf5044eb33086b2a84961610fec15b461
+        checksum/agent-windows: 9580d68fcd452c53eb53272cc077b07295505b7209185d3e36619fb2f02fb935
         checksum/windows-config: 6ff4f8bd0b310ebe4d4612bdd9697ffb3d79e0e0eab3936420417dd5a8fc128d
         microsoft.com/hostprocess-inherit-user: "true"
       labels:

build/yamls/windows/containerd-with-ovs/conf/Run-AntreaOVS-Containerd.ps1

@@ -2,16 +2,31 @@ $ErrorActionPreference = "Stop"
 $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
 $mountPath = ($mountPath.Replace('\', '/')).TrimEnd('/')
 $env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/openvswitch/usr/bin;$mountPath/openvswitch/usr/sbin"
+$OVSDriverDir="$mountPath\openvswitch\driver"
+
+# Check if OVSExt driver is already installed
+$driverStatus = netcfg -q ovsext
+if ($driverStatus -like '*not installed*') {
+  # Install OVS Driver
+  netcfg -l $OVSDriverDir/ovsext.inf -c s -i OVSExt
+}
+
+# Configure OVS processes
 $OVS_DB_SCHEMA_PATH = "$mountPath/openvswitch/usr/share/openvswitch/vswitch.ovsschema"
 $OVS_DB_PATH = "C:\openvswitch\etc\openvswitch\conf.db"
 if ($(Test-Path $OVS_DB_SCHEMA_PATH) -and !$(Test-Path $OVS_DB_PATH)) {
+    mkdir -force C:\openvswitch\etc\openvswitch\
     ovsdb-tool create "$OVS_DB_PATH" "$OVS_DB_SCHEMA_PATH"
 }
+$OVS_RUN_PATH = "C:\openvswitch\var\run\openvswitch"
+if (!$(Test-Path $OVS_RUN_PATH)) {
+  mkdir -force $OVS_RUN_PATH
+}
 ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach
 ovs-vsctl --no-wait init
 
 # Set OVS version.
-$OVS_VERSION=$(Get-Item $mountPath\openvswitch\driver\OVSExt.sys).VersionInfo.ProductVersion
+$OVS_VERSION=$(Get-Item $OVSDriverDir\OVSExt.sys).VersionInfo.ProductVersion
 ovs-vsctl --no-wait set Open_vSwitch . ovs_version=$OVS_VERSION
 
 ovs-vswitchd --log-file=/var/log/antrea/openvswitch/ovs-vswitchd.log --pidfile -vfile:info --detach

ci/jenkins/test.sh

@@ -152,7 +152,12 @@ export NO_PULL
 E2ETEST_PATH=${WORKDIR}/kubernetes/_output/dockerized/bin/linux/amd64/e2e.test
 
 function export_govc_env_var {
-    export GOVC_URL=$GOVC_URL
+    env_govc="${WORKDIR}/govc.env"
+    if [ -f "$env_govc" ]; then
+        source "$env_govc"
+    else
+        export GOVC_URL=$GOVC_URL
+    fi
     export GOVC_USERNAME=$GOVC_USERNAME
     export GOVC_PASSWORD=$GOVC_PASSWORD
     export GOVC_INSECURE=1
@@ -322,7 +327,11 @@ function prepare_env {
 function revert_snapshot_windows {
     WIN_NAME=$1
     echo "==== Reverting Windows VM ${WIN_NAME} ====="
-    govc snapshot.revert -vm ${WIN_NAME} win-initial
+    if [[ $WIN_NAME == *"jumper"* ]]; then
+        govc snapshot.revert -vm ${WIN_NAME} win-initial
+    else
+        govc snapshot.revert -vm ${WIN_NAME} pristine-win-initial
+    fi
     # If Windows VM fails to power on correctly in time, retry several times.
     winVMIPs=""
     for i in `seq 10`; do
@@ -516,26 +525,24 @@ function build_and_deliver_antrea_windows_and_linux_docker_images {
 function  build_and_deliver_antrea_windows_and_linux_containerd_images {
     echo "====== Cleanup Antrea Installation Before Delivering Antrea Windows and Antrea Linux containerd Images ====="
     clean_antrea
-    kubectl delete -f ${WORKDIR}/antrea-windows-containerd.yml --ignore-not-found=true || true
+    kubectl delete -f ${WORKDIR}/antrea-windows-containerd-with-ovs.yml --ignore-not-found=true || true
     kubectl delete -f ${WORKDIR}/kube-proxy-windows-containerd.yml --ignore-not-found=true || true
     kubectl delete daemonset antrea-agent -n kube-system --ignore-not-found=true || true
     kubectl delete -f ${WORKDIR}/antrea.yml --ignore-not-found=true || true
 
     prepare_env
-    ${CLEAN_STALE_IMAGES_CONTAINERD}
-    ${PRINT_CONTAINERD_STATUS}
     chmod -R g-w build/images/ovs
     chmod -R g-w build/images/base
     # Clean docker image to save disk space.
     ${CLEAN_STALE_IMAGES}
     ${PRINT_DOCKER_STATUS}
     export_govc_env_var
     # Enable verbose log for troubleshooting.
-    sed -i "s/--v=0/--v=4/g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd.yml
+    sed -i "s/--v=0/--v=4/g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd-with-ovs.yml
 
     echo "====== Updating yaml files to enable proxyAll ======"
     KUBE_API_SERVER=$(kubectl --kubeconfig=$KubeConfigFile config view -o jsonpath='{.clusters[0].cluster.server}')
-    sed -i "s|.*kubeAPIServerOverride: \"\"|    kubeAPIServerOverride: \"${KUBE_API_SERVER}\"|g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd.yml
+    sed -i "s|.*kubeAPIServerOverride: \"\"|    kubeAPIServerOverride: \"${KUBE_API_SERVER}\"|g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd-with-ovs.yml
 
     cp -f build/yamls/*.yml $WORKDIR
     set +e
@@ -1148,7 +1155,7 @@ fi
 trap clean_antrea EXIT
 if [[ ${TESTCASE} =~ "windows" ]]; then
     if [[ ${TESTCASE} =~ "containerd" ]]; then
-        WINDOWS_YAML_SUFFIX="windows-containerd"
+        WINDOWS_YAML_SUFFIX="windows-containerd-with-ovs"
         build_and_deliver_antrea_windows_and_linux_containerd_images
         if [[ ${TESTCASE} =~ "e2e" ]]; then
             run_e2e_windows

docs/windows.md

@@ -222,12 +222,25 @@ kubectl apply -f -
 
 ##### 1. (Optional) Install OVS (provided by Antrea or your own)
 
+Antrea supports running OVS on Windows as native services or inside a
+host-process container. If you have an OVS package with a signed kernel
+driver and want to run OVS inside container, you can skip this step.
+
 Antrea provides a pre-built OVS package which contains test-signed OVS kernel
 driver. If you don't have a self-signed OVS package and just want to try the
 Antrea on Windows, this package can be used for testing. We also provide a helper
 script `Install-OVS.ps1` to install the OVS driver and register userspace binaries
-as services. If you want to use your own signed OVS package for production, you can
-run `Install-OVS.ps1` like this:
+as services. 
+
+If you want to containerize OVS with an unsigned kernel driver, you must
+pre-install the driver on the worker node before joining cluster. Hence, 
+you need to run the `Install-OVS.ps1` script to install only the driver like this:
+
+```powershell
+.\Install-OVS.ps1 -InstallUserspace $false
+```
+
+If you want to run OVS as Windows native services, you can run the script like this,
 
 ```powershell
 Install-OVS.ps1 -ImportCertificate $false -Local -LocalFile <PathToOVSPackage>
@@ -256,13 +269,6 @@ get-service ovsdb-server
 get-service ovs-vswitchd
 ```
 
-If you want to containerize OVS for containerd runtime, OVS userspace processes are
-not run on the host and hence you can set the `InstallUserspace` parameter to false.
-
-```powershell
-.\Install-OVS.ps1 -InstallUserspace $false
-```
-
 ##### 2. Disable Windows Firewall
 
 ```powershell
@@ -340,6 +346,16 @@ The script `Prepare-AntreaAgent.ps1` performs following tasks:
     After the Windows Node reboots, there will be stale network resources which
     need to be cleaned before starting antrea-agent.
 
+* Ensure OVS services are running.
+
+    This script starts OVS services on the Node if they are not running. This step
+    needs to be skipped in case of OVS containerization. Hence, you need to specify
+    the parameter `RunOVSServices` as false.
+
+    ```powershell
+    & C:\k\antrea\Prepare-AntreaAgent.ps1 -RunOVSServices $false
+    ```
+
 As you know from the task details from above, the script must be executed every
 time you restart the Node to prepare the environment for antrea-agent.
 

hack/windows/Prepare-AntreaAgent.ps1

@@ -13,7 +13,8 @@ Specifies whether kube-proxy interface is included in the installation. If false
 be installed on the host.
 #>
 Param(
-    [parameter(Mandatory = $false)] [bool] $InstallKubeProxy = $true
+    [parameter(Mandatory = $false)] [bool] $InstallKubeProxy = $true,
+    [parameter(Mandatory = $false)] [bool] $RunOVSServices= $true
 )
 
 $ErrorActionPreference = 'Stop'
@@ -35,10 +36,12 @@ if ($NeedCleanNetwork) {
     & $CleanAntreaNetworkScript
 }
 # Enure OVS services are running.
-Write-Host "Starting ovsdb-server service..."
-Start-Service ovsdb-server
-Write-Host "Starting ovs-vswitchd service..."
-Start-Service ovs-vswitchd
+if ($RunOVSServices -eq $true) {
+    Write-Host "Starting ovsdb-server service..."
+    Start-Service ovsdb-server
+    Write-Host "Starting ovs-vswitchd service..."
+    Start-Service ovs-vswitchd
+}
 # Prepare service network interface for kube-proxy.
 if ($InstallKubeProxy -eq $true) {
     Write-Host "Preparing service network interface for kube-proxy..."