[Windows] Support run antrea-agent directly from scripts (#1013)

Currently Antrea runs antrea-agent and kube-proxy from management
Pods. But there is still need to run these two components directly
using scripts. This patch:
- Adds scripts help to install and run antrea-agent and kube-proxy.
- Support read kubeconfig from file for antrea-agent apiserver.
  Which is needed when antrea-agent is running as process.

Signed-off-by: Rui Cao <rcao@vmware.com>

cmd/antrea-agent/agent.go

@@ -225,7 +225,8 @@ func run(o *Options) error {
 		agentQuerier,
 		networkPolicyController,
 		o.config.APIPort,
-		o.config.EnablePrometheusMetrics)
+		o.config.EnablePrometheusMetrics,
+		o.config.ClientConnection.Kubeconfig)
 	if err != nil {
 		return fmt.Errorf("error when creating agent API server: %v", err)
 	}

docs/windows.md

@@ -21,6 +21,10 @@ The following components should be configured and run on the Windows Node.
 
 antrea-agent and kube-proxy run as processes on host and are managed by
 management Pods. It is recommended to run OVS daemons as Windows services.
+If you don't want to run antrea-agent and kube-proxy from management Pods
+Antrea also provide scripts which help install and run these two components
+directlty without Pod, please see [Manually run kube-proxy and antrea-agent on Windows worker Nodes](#Manually-run-kube-proxy-and-antrea-agent-on-Windows-worker-Nodes)
+section for details.
 
 ## Deploying Antrea on Windows Worker Node
 
@@ -219,7 +223,24 @@ kubectl get pods -o wide -nkube-system | grep windows
 antrea-agent-windows-6hvkw                             1/1     Running     0          100s
 kube-proxy-windows-2d45w                               1/1     Running     0          102s
 ```
+### Manually run kube-proxy and antrea-agent on Windows worker Nodes
 
+Besides start kube-proxy and antrea-agent from management Pods, Antrea also
+provides powershell scripts which help install and run these two components
+directly without Pod. Please complete the steps in [Installation](#Installation)
+section, skip [Add Windows kube-proxy DaemonSet](#Add-Windows-kube-proxy-DaemonSet)
+and [Add Windows antrea-agent DaemonSet](#Add-Windows-antrea-agent-DaemonSet)
+stpes. And then run following commands in powershell.
+```
+mkdir c:\k\antrea
+cd c:\k\antrea
+curl.exe -LO https://raw.githubusercontent.com/vmware-tanzu/antrea/master/hack/windows/Start.ps1
+./Start.ps1
+``` 
+
+Please skip [Add Windows kube-proxy DaemonSet](#Add-Windows-kube-proxy-DaemonSet)
+and [Add Windows antrea-agent DaemonSet](#Add-Windows-antrea-agent-DaemonSet)
+sections if you choose manually run kube-proxy and antrea-agent.
 
 ## Known issues
 1. HNS Network is not persistent on Windows. So after the Windows Node reboots,

hack/windows/Helper.psm1

@@ -0,0 +1,37 @@
+function DownloadFileIfNotExist($Path, $URL) {
+    if (Test-Path $Path) {
+        return
+    }
+    Write-Host "Downloading $URL to $PATH"
+    curl.exe -sLo $Path $URL
+}
+
+function CreateDirectory($Path)
+{
+    if (!(Test-Path $Path))
+    {
+        mkdir $Path
+    }
+}
+
+Function TestConnection($ComputerName, $Port, $MaxRetry, $Interval) {
+    $RetryCountRange = 1..$MaxRetry
+    foreach ($RetryCount in $RetryCountRange) {
+        Write-Host "Testing connection to ($ComputerName,$Port) ($RetryCount/$MaxRetry)..."
+        if (Test-NetConnection -ComputerName $ComputerName -Port $Port | ? { $_.TcpTestSucceeded }) {
+            return $true
+        }
+        if ($RetryCount -eq $MaxRetry) {
+            return $false
+        }
+        Start-Sleep -Seconds $Interval
+    }
+}
+
+Function GetGithubLatestReleaseTag($Owner, $Repo) {
+    return (curl.exe -s "https://api.github.com/repos/$Owner/$Repo/releases" | ConvertFrom-Json)[0].tag_name
+}
+
+Export-ModuleMember DownloadFileIfNotExist
+Export-ModuleMember CreateDirectory
+Export-ModuleMember TestConnection

hack/windows/Install-AntreaAgent.ps1

@@ -0,0 +1,100 @@
+Param(
+    [parameter(Mandatory = $false, HelpMessage="Kubernetes version to use")] [string] $KubernetesVersion="v1.18.0",
+    [parameter(Mandatory = $false, HelpMessage="Kubernetes home path")] [string] $KubernetesHome="c:\k",
+    [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config",
+    [parameter(Mandatory = $false, HelpMessage="Antrea version to use")] [string] $AntreaVersion="latest",
+    [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea"
+)
+$ErrorActionPreference = "Stop"
+
+
+$kubectl = "$KubernetesHome\kubectl.exe"
+$KubeProxy = "$KubernetesHome\kube-proxy.exe"
+$yq = "$KubernetesHome\yq.exe"
+
+$CNIPath = "c:\opt\cni\bin"
+$CNIConfigPath = "c:\etc\cni\net.d"
+$AntreaCNIConfigFile = "$CNIPath\10-antrea.conflist"
+$HostLocalIpam = "$CNIPath\host-local.exe"
+
+$AntreaEtc = "$AntreaHome\etc"
+$AntreaAgentConfigPath = "$AntreaEtc\antrea-agent.conf"
+$AntreaAgent = "$AntreaHome\bin\antrea-agent.exe"
+$AntreaCNI = "$CNIPath\antrea.exe"
+$PrepareServiceInterface = "$AntreaHome\Prepare-ServiceInterface.ps1"
+$StartKubeProxy = "$AntreaHome\Start-KubeProxy.ps1"
+$StartAntreaAgent = "$AntreaHome\Start-AntreaAgent.ps1"
+$StopScript = "$AntreaHome\Stop.ps1"
+$Owner = "ruicao93"
+$Repo = "antrea"
+
+$env:Path = "$KubernetesHome;" + $env:Path
+$helper = "$AntreaHome\Helper.psm1"
+Import-Module $helper
+
+if ($AntreaVersion -eq "latest") {
+    $AntreaVersion = (GetGithubLatestReleaseTag $Owner $Repo)
+}
+Write-Host "Antrea version: $AntreaVersion"
+$AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion"
+$AntreaReleaseUrlBase = "https://github.com/$Owner/$Repo/releases/download"
+$AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion"
+
+
+CreateDirectory $KubernetesHome
+# Download kubectl
+DownloadFileIfNotExist $kubectl  "https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kubectl.exe"
+# Download kube-proxy
+DownloadFileIfNotExist $KubeProxy "https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kube-proxy.exe"
+# Download yq
+DownloadFileIfNotExist $yq "https://github.com/mikefarah/yq/releases/download/3.3.2/yq_windows_amd64.exe"
+
+CreateDirectory $AntreaHome
+CreateDirectory "$AntreaHome\bin"
+CreateDirectory "$CNIPath"
+CreateDirectory "$CNIConfigPath"
+# Download antrea-agent for windows
+DownloadFileIfNotExist $AntreaAgent  "$AntreaReleaseUrlBase/$AntreaVersion/antrea-agent-windows-x86_64.exe"
+DownloadFileIfNotExist $AntreaCNI  "$AntreaReleaseUrlBase/$AntreaVersion/antrea-cni-windows-x86_64.exe"
+# Prepare antrea scripts
+DownloadFileIfNotExist $PrepareServiceInterface  "$AntreaRawUrlBase/hack/windows/Prepare-ServiceInterface.ps1"
+DownloadFileIfNotExist $StartKubeProxy  "$AntreaRawUrlBase/hack/windows/Start-KubeProxy.ps1"
+DownloadFileIfNotExist $StartAntreaAgent  "$AntreaRawUrlBase/hack/windows/Start-AntreaAgent.ps1"
+DownloadFileIfNotExist $StopScript  "$AntreaRawUrlBase/hack/windows/Stop.ps1"
+
+# Download host-local IPAM plugin
+if (!(Test-Path $HostLocalIpam)) {
+    curl.exe -sLO https://github.com/containernetworking/plugins/releases/download/v0.8.1/cni-plugins-windows-amd64-v0.8.1.tgz
+    C:\Windows\system32\tar.exe -xzf cni-plugins-windows-amd64-v0.8.1.tgz  -C $CNIPath "./host-local.exe"
+    Remove-Item cni-plugins-windows-amd64-v0.8.1.tgz
+}
+
+CreateDirectory $AntreaEtc
+DownloadFileIfNotExist $AntreaCNIConfigFile "$AntreaRawUrlBase/build/yamls/windows/base/conf/antrea-cni.conflist"
+if (!(Test-Path $AntreaAgentConfigPath)) {
+    DownloadFileIfNotExist $AntreaAgentConfigPath "$AntreaRawUrlBase/build/yamls/windows/base/conf/antrea-agent.conf"
+    yq w -i $AntreaAgentConfigPath featureGates.AntreaProxy true
+    yq w -i $AntreaAgentConfigPath clientConnection.kubeconfig $AntreaEtc\antrea-agent.kubeconfig
+    yq w -i $AntreaAgentConfigPath antreaClientConnection.kubeconfig $AntreaEtc\antrea-agent.antrea.kubeconfig
+    $env:kubeconfig = $KubeConfig
+    # Create the kubeconfig file that contains the K8s APIServer endpoint and the token of antrea ServiceAccount.
+    $APIServer=$(kubectl get service kubernetes -o jsonpath='{.spec.clusterIP}')
+    $APIServerPort=$(kubectl get service kubernetes -o jsonpath='{.spec.ports[0].port}')
+    $APIServer="https://$APIServer" + ":" + $APIServerPort
+    $TOKEN=$(kubectl get secrets -n kube-system -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='antrea-agent')].data.token}")
+    $TOKEN=$([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($TOKEN)))
+    kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-cluster kubernetes --server=$APIServer --insecure-skip-tls-verify
+    kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-credentials antrea-agent --token=$TOKEN
+    kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-context antrea-agent@kubernetes --cluster=kubernetes --user=antrea-agent
+    kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig use-context antrea-agent@kubernetes
+
+    $AntreaAPISServer=$(kubectl get service -n kube-system antrea -o jsonpath='{.spec.clusterIP}')
+    $AntreaAPISServerPort=$(kubectl get service -n kube-system antrea -o jsonpath='{.spec.ports[0].port}')
+    $AntreaAPISServer="https://$AntreaAPISServer" + ":" + $AntreaAPISServerPort
+    $TOKEN=$(kubectl get secrets -n kube-system -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='antrea-agent')].data.token}")
+    $TOKEN=$([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($TOKEN)))
+    kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-cluster antrea --server=$AntreaAPISServer --insecure-skip-tls-verify
+    kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-credentials antrea-agent --token=$TOKEN
+    kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-context antrea-agent@antrea --cluster=antrea --user=antrea-agent
+    kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig use-context antrea-agent@antrea
+}

hack/windows/Start-AntreaAgent.ps1

@@ -0,0 +1,22 @@
+Param(
+    [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea",
+    [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config",
+    [parameter(Mandatory = $false)] [string] $LogDir
+)
+$ErrorActionPreference = "Stop"
+
+if (Get-Process -Name antrea-agent -ErrorAction SilentlyContinue) {
+    Write-Host "antrea-agent is already in running, exit"
+    exit 0
+}
+
+Import-Module c:\k\antrea\Helper.psm1
+$AntreaAgent = "$AntreaHome\bin\antrea-agent.exe"
+$AntreaAgentConfigPath = "$AntreaHome\etc\antrea-agent.conf"
+if ($LogDir -eq "") {
+    $LogDir = "$AntreaHome\logs"
+}
+
+CreateDirectory $LogDir
+[Environment]::SetEnvironmentVariable("NODE_NAME", (hostname).ToLower())
+& $AntreaAgent  --config=$AntreaAgentConfigPath --logtostderr=false --log_dir=$LogDir --alsologtostderr --log_file_max_size=100 --log_file_max_num=4

hack/windows/Start-KubeProxy.ps1

@@ -0,0 +1,25 @@
+Param(
+    [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeProxy = "c:\k\kube-proxy.exe",
+    [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config",
+    [parameter(Mandatory = $false)] [string] $LogDir = "c:\var\log\kube-proxy"
+)
+$ErrorActionPreference = "Stop"
+
+$PrepareServiceInterface = "c:\k\antrea\Prepare-ServiceInterface.ps1"
+Import-Module c:\k\antrea\Helper.psm1
+
+if (Get-Process -Name kube-proxy -ErrorAction SilentlyContinue) {
+    Write-Host "kube-proxy is already in running, exit"
+    exit 0
+}
+
+CreateDirectory $LogDir
+
+powershell $PrepareServiceInterface
+
+if ($LastExitCode) {
+    Write-Host "Prepare kube-proxy service interface failed, exit"
+    exit 1
+}
+
+& $KubeProxy --proxy-mode=userspace --kubeconfig=$KubeConfig --log-dir=$LogDir --logtostderr=false --alsologtostderr --v=4

hack/windows/Start.ps1

@@ -0,0 +1,59 @@
+Param(
+    [parameter(Mandatory = $false, HelpMessage="Kubernetes version to use")] [string] $KubernetesVersion="v1.18.0",
+    [parameter(Mandatory = $false, HelpMessage="Kubernetes home path")] [string] $KubernetesHome="c:\k",
+    [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config",
+    [parameter(Mandatory = $false, HelpMessage="Antrea version to use")] [string] $AntreaVersion="latest",
+    [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea"
+)
+$ErrorActionPreference = "Stop"
+
+$InstallAntreaAgent = "$AntreaHome\Install-AntreaAgent.ps1"
+$Owner = "ruicao93"
+$Repo = "antrea"
+if ($AntreaVersion -eq "latest") {
+    $AntreaVersion = (curl.exe -s "https://api.github.com/repos/$Owner/$Repo/releases" | ConvertFrom-Json)[0].tag_name
+}
+Write-Host "KubernetesVersion version: $KubernetesVersion"
+Write-Host "Antrea version: $AntreaVersion"
+$AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion"
+
+if (!(Test-Path $AntreaHome)) {
+    mkdir $AntreaHome
+}
+
+$helper = "$AntreaHome\Helper.psm1"
+if (!(Test-Path $helper))
+{
+    curl.exe -sLo $helper "$AntreaRawUrlBase/hack/windows/Helper.psm1"
+}
+Import-Module $helper
+
+# Download Install-AntreaAgent.ps1
+DownloadFileIfNotExist $InstallAntreaAgent  "$AntreaRawUrlBase/hack/windows/Install-AntreaAgent.ps1"
+
+Write-Host "Checking kube-proxy and antrea-agent installation..."
+powershell $InstallAntreaAgent -KubernetesVersion $KubernetesVersion -KubernetesHome $KubernetesHome -KubeConfig $KubeConfig -AntreaVersion $AntreaVersion -AntreaHome $AntreaHome
+
+if ($LastExitCode) {
+    Write-Host "Install antrea-agent failed, exit"
+    exit 1
+}
+
+Write-Host "Starting kube-proxy..."
+Start-Process powershell -ArgumentList "-File $AntreaHome\Start-KubeProxy.ps1 -KubeProxy $KubernetesHome\kube-proxy.exe -KubeConfig $KubeConfig"
+
+$env:kubeconfig = $KubeConfig
+$APIServer=$(kubectl get service kubernetes -o jsonpath='{.spec.clusterIP}')
+$APIServerPort=$(kubectl get service kubernetes -o jsonpath='{.spec.ports[0].port}')
+$APIServer="https://$APIServer" + ":" + $APIServerPort
+$APIServer=[System.Uri]$APIServer
+
+Write-Host "Test connection to kubernetes API server"
+$result = TestConnection $APIServer.Host $APIServer.Port 20 3
+if (!$result) {
+    Write-Host "Failed to connection to kubernetes API server service, exit"
+    exit 1
+}
+
+Write-Host "Starting antrea-agent..."
+Start-Process powershell -ArgumentList "-File $AntreaHome\Start-AntreaAgent.ps1 -AntreaHome $AntreaHome -KubeConfig $KubeConfig"

hack/windows/Stop.ps1

@@ -0,0 +1,2 @@
+taskkill /im antrea-agent.exe /f
+taskkill /im kube-proxy.exe /f

pkg/agent/apiserver/apiserver.go

@@ -89,8 +89,8 @@ func installAPIGroup(s *genericapiserver.GenericAPIServer, aq agentquerier.Agent
 
 // New creates an APIServer for running in antrea agent.
 func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier, bindPort int,
-	enableMetrics bool) (*agentAPIServer, error) {
-	cfg, err := newConfig(bindPort, enableMetrics)
+	enableMetrics bool, kubeconfig string) (*agentAPIServer, error) {
+	cfg, err := newConfig(bindPort, enableMetrics, kubeconfig)
 	if err != nil {
 		return nil, err
 	}
@@ -105,11 +105,17 @@ func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier
 	return &agentAPIServer{GenericAPIServer: s}, nil
 }
 
-func newConfig(bindPort int, enableMetrics bool) (*genericapiserver.CompletedConfig, error) {
+func newConfig(bindPort int, enableMetrics bool, kubeconfig string) (*genericapiserver.CompletedConfig, error) {
 	secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
 	authentication := genericoptions.NewDelegatingAuthenticationOptions()
 	authorization := genericoptions.NewDelegatingAuthorizationOptions().WithAlwaysAllowPaths("/healthz")
 
+	// kubeconfig file is useful when antrea-agent isn't not running as a pod
+	if len(kubeconfig) > 0 {
+		authentication.RemoteKubeConfigFile = kubeconfig
+		authorization.RemoteKubeConfigFile = kubeconfig
+	}
+
 	// Set the PairName but leave certificate directory blank to generate in-memory by default.
 	secureServing.ServerCert.CertDirectory = ""
 	secureServing.ServerCert.PairName = Name