From 5dccb784ef9f94a237500147d84366f4086bce83 Mon Sep 17 00:00:00 2001 From: Rui Cao Date: Thu, 30 Jul 2020 16:09:44 +0800 Subject: [PATCH] [Windows] Support run antrea-agent directly from scripts (#1013) Currently Antrea runs antrea-agent and kube-proxy from management Pods. But there is still need to run these two components directly using scripts. This patch: - Adds scripts help to install and run antrea-agent and kube-proxy. - Support read kubeconfig from file for antrea-agent apiserver. Which is needed when antrea-agent is running as process. Signed-off-by: Rui Cao --- cmd/antrea-agent/agent.go | 3 +- docs/windows.md | 21 +++ hack/windows/Helper.psm1 | 223 +++++++++++++++++++++++++++++++ hack/windows/Start.ps1 | 63 +++++++++ hack/windows/Stop.ps1 | 2 + pkg/agent/apiserver/apiserver.go | 12 +- 6 files changed, 320 insertions(+), 4 deletions(-) create mode 100644 hack/windows/Helper.psm1 create mode 100644 hack/windows/Start.ps1 create mode 100644 hack/windows/Stop.ps1 diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 2dd9386f02e..c48f0252354 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -225,7 +225,8 @@ func run(o *Options) error { agentQuerier, networkPolicyController, o.config.APIPort, - o.config.EnablePrometheusMetrics) + o.config.EnablePrometheusMetrics, + o.config.ClientConnection.Kubeconfig) if err != nil { return fmt.Errorf("error when creating agent API server: %v", err) } diff --git a/docs/windows.md b/docs/windows.md index d0da7596192..3c03d5b572b 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -21,6 +21,10 @@ The following components should be configured and run on the Windows Node. antrea-agent and kube-proxy run as processes on host and are managed by management Pods. It is recommended to run OVS daemons as Windows services. +If you don't want to run antrea-agent and kube-proxy from management Pods +Antrea also provide scripts which help install and run these two components +directlty without Pod, please see [Manually run kube-proxy and antrea-agent on Windows worker Nodes](#Manually-run-kube-proxy-and-antrea-agent-on-Windows-worker-Nodes) +section for details. ## Deploying Antrea on Windows Worker Node @@ -219,7 +223,24 @@ kubectl get pods -o wide -nkube-system | grep windows antrea-agent-windows-6hvkw 1/1 Running 0 100s kube-proxy-windows-2d45w 1/1 Running 0 102s ``` +### Manually run kube-proxy and antrea-agent on Windows worker Nodes +Besides start kube-proxy and antrea-agent from management Pods, Antrea also +provides powershell scripts which help install and run these two components +directly without Pod. Please complete the steps in [Installation](#Installation) +section, skip [Add Windows kube-proxy DaemonSet](#Add-Windows-kube-proxy-DaemonSet) +and [Add Windows antrea-agent DaemonSet](#Add-Windows-antrea-agent-DaemonSet) +stpes. And then run following commands in powershell. +``` +mkdir c:\k\antrea +cd c:\k\antrea +curl.exe -LO https://raw.githubusercontent.com/vmware-tanzu/antrea/master/hack/windows/Start.ps1 +./Start.ps1 +``` + +Please skip [Add Windows kube-proxy DaemonSet](#Add-Windows-kube-proxy-DaemonSet) +and [Add Windows antrea-agent DaemonSet](#Add-Windows-antrea-agent-DaemonSet) +sections if you choose manually run kube-proxy and antrea-agent. ## Known issues 1. HNS Network is not persistent on Windows. So after the Windows Node reboots, diff --git a/hack/windows/Helper.psm1 b/hack/windows/Helper.psm1 new file mode 100644 index 00000000000..5c498ece40c --- /dev/null +++ b/hack/windows/Helper.psm1 @@ -0,0 +1,223 @@ +function Get-WebFileIfNotExist($Path, $URL) { + if (Test-Path $Path) { + return + } + Write-Host "Downloading $URL to $PATH" + curl.exe -sLo $Path $URL +} + +function New-DirectoryIfNotExist($Path) +{ + if (!(Test-Path $Path)) + { + mkdir $Path + } +} + +function Test-ConnectionWithRetry($ComputerName, $Port, $MaxRetry, $Interval) { + $RetryCountRange = 1..$MaxRetry + foreach ($RetryCount in $RetryCountRange) { + Write-Host "Testing connection to ($ComputerName,$Port) ($RetryCount/$MaxRetry)..." + if (Test-NetConnection -ComputerName $ComputerName -Port $Port | ? { $_.TcpTestSucceeded }) { + return $true + } + if ($RetryCount -eq $MaxRetry) { + return $false + } + Start-Sleep -Seconds $Interval + } +} + +function Get-GithubLatestReleaseTag($Owner, $Repo) { + return (curl.exe -s "https://api.github.com/repos/$Owner/$Repo/releases" | ConvertFrom-Json)[0].tag_name +} + +function Install-AntreaAgent { + Param( + [parameter(Mandatory = $false, HelpMessage="Kubernetes version to use")] [string] $KubernetesVersion="v1.18.0", + [parameter(Mandatory = $false, HelpMessage="Kubernetes home path")] [string] $KubernetesHome="c:\k", + [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config", + [parameter(Mandatory = $false, HelpMessage="Antrea version to use")] [string] $AntreaVersion="latest", + [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea" + ) + $ErrorActionPreference = "Stop" + + $kubectl = "$KubernetesHome\kubectl.exe" + $KubeProxy = "$KubernetesHome\kube-proxy.exe" + $yq = "$KubernetesHome\yq.exe" + + $CNIPath = "c:\opt\cni\bin" + $CNIConfigPath = "c:\etc\cni\net.d" + $AntreaCNIConfigFile = "$CNIConfigPath\10-antrea.conflist" + $HostLocalIpam = "$CNIPath\host-local.exe" + + $AntreaEtc = "$AntreaHome\etc" + $AntreaAgentConfigPath = "$AntreaEtc\antrea-agent.conf" + $AntreaAgent = "$AntreaHome\bin\antrea-agent.exe" + $AntreaCNI = "$CNIPath\antrea.exe" + $StopScript = "$AntreaHome\Stop.ps1" + $Owner = "vmware-tanzu" + $Repo = "antrea" + + $env:Path = "$KubernetesHome;" + $env:Path + + if ($AntreaVersion -eq "latest") { + $AntreaVersion = (Get-GithubLatestReleaseTag $Owner $Repo) + } + Write-Host "Antrea version: $AntreaVersion" + $AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion" + $AntreaReleaseUrlBase = "https://github.com/$Owner/$Repo/releases/download" + $AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion" + + + New-DirectoryIfNotExist $KubernetesHome + # Download kubectl + Get-WebFileIfNotExist $kubectl "https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kubectl.exe" + # Download kube-proxy + Get-WebFileIfNotExist $KubeProxy "https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kube-proxy.exe" + # Download yq + Get-WebFileIfNotExist $yq "https://github.com/mikefarah/yq/releases/download/3.3.2/yq_windows_amd64.exe" + + New-DirectoryIfNotExist $AntreaHome + New-DirectoryIfNotExist "$AntreaHome\bin" + New-DirectoryIfNotExist "$CNIPath" + New-DirectoryIfNotExist "$CNIConfigPath" + # Download antrea-agent for windows + Get-WebFileIfNotExist $AntreaAgent "$AntreaReleaseUrlBase/$AntreaVersion/antrea-agent-windows-x86_64.exe" + Get-WebFileIfNotExist $AntreaCNI "$AntreaReleaseUrlBase/$AntreaVersion/antrea-cni-windows-x86_64.exe" + # Prepare antrea scripts + Get-WebFileIfNotExist $StopScript "$AntreaRawUrlBase/hack/windows/Stop.ps1" + + # Download host-local IPAM plugin + if (!(Test-Path $HostLocalIpam)) { + curl.exe -sLO https://github.com/containernetworking/plugins/releases/download/v0.8.1/cni-plugins-windows-amd64-v0.8.1.tgz + C:\Windows\system32\tar.exe -xzf cni-plugins-windows-amd64-v0.8.1.tgz -C $CNIPath "./host-local.exe" + Remove-Item cni-plugins-windows-amd64-v0.8.1.tgz + } + + New-DirectoryIfNotExist $AntreaEtc + Get-WebFileIfNotExist $AntreaCNIConfigFile "$AntreaRawUrlBase/build/yamls/windows/base/conf/antrea-cni.conflist" + if (!(Test-Path $AntreaAgentConfigPath)) { + Get-WebFileIfNotExist $AntreaAgentConfigPath "$AntreaRawUrlBase/build/yamls/windows/base/conf/antrea-agent.conf" + yq w -i $AntreaAgentConfigPath featureGates.AntreaProxy true + yq w -i $AntreaAgentConfigPath clientConnection.kubeconfig $AntreaEtc\antrea-agent.kubeconfig + yq w -i $AntreaAgentConfigPath antreaClientConnection.kubeconfig $AntreaEtc\antrea-agent.antrea.kubeconfig + $env:kubeconfig = $KubeConfig + # Create the kubeconfig file that contains the K8s APIServer service and the token of antrea ServiceAccount. + $APIServer=$(kubectl get service kubernetes -o jsonpath='{.spec.clusterIP}') + $APIServerPort=$(kubectl get service kubernetes -o jsonpath='{.spec.ports[0].port}') + $APIServer="https://$APIServer" + ":" + $APIServerPort + $TOKEN=$(kubectl get secrets -n kube-system -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='antrea-agent')].data.token}") + $TOKEN=$([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($TOKEN))) + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-cluster kubernetes --server=$APIServer --insecure-skip-tls-verify + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-credentials antrea-agent --token=$TOKEN + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-context antrea-agent@kubernetes --cluster=kubernetes --user=antrea-agent + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig use-context antrea-agent@kubernetes + + # Create the kubeconfig file that contains the antrea-controller APIServer service and the token of antrea ServiceAccount. + $AntreaAPISServer=$(kubectl get service -n kube-system antrea -o jsonpath='{.spec.clusterIP}') + $AntreaAPISServerPort=$(kubectl get service -n kube-system antrea -o jsonpath='{.spec.ports[0].port}') + $AntreaAPISServer="https://$AntreaAPISServer" + ":" + $AntreaAPISServerPort + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-cluster antrea --server=$AntreaAPISServer --insecure-skip-tls-verify + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-credentials antrea-agent --token=$TOKEN + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-context antrea-agent@antrea --cluster=antrea --user=antrea-agent + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig use-context antrea-agent@antrea + } +} + +function New-KubeProxyServiceInterface { + Param( + [parameter(Mandatory = $false, HelpMessage="Interface to be added service IPs by kube-proxy")] [string] $InterfaceAlias="HNS Internal NIC" + ) + $ErrorActionPreference = "Stop" + + $hnsSwitchName = "KubeProxyInternalSwitch" + $INTERFACE_TO_ADD_SERVICE_IP = "vEthernet ($InterfaceAlias)" + if (Get-NetAdapter -InterfaceAlias $INTERFACE_TO_ADD_SERVICE_IP -ErrorAction SilentlyContinue) { + Write-Host "Network adapter $INTERFACE_TO_ADD_SERVICE_IP exists, exit." + return + } + if (!(Get-VMSwitch -Name $hnsSwitchName -ErrorAction SilentlyContinue)) { + Write-Host "Creating internal switch: $hnsSwitchName for kube-proxy" + New-VMSwitch -name $hnsSwitchName -SwitchType Internal + } + Write-Host "Creating network adapter: $INTERFACE_TO_ADD_SERVICE_IP for kube-proxy" + [Environment]::SetEnvironmentVariable("INTERFACE_TO_ADD_SERVICE_IP", $INTERFACE_TO_ADD_SERVICE_IP, [System.EnvironmentVariableTarget]::Machine) + Add-VMNetworkAdapter -ManagementOS -Name $InterfaceAlias -SwitchName $hnsSwitchName + Set-NetIPInterface -ifAlias $INTERFACE_TO_ADD_SERVICE_IP -Forwarding Enabled +} + +function Start-KubeProxy { + Param( + [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeProxy = "c:\k\kube-proxy.exe", + [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config", + [parameter(Mandatory = $false)] [string] $LogDir = "c:\var\log\kube-proxy" + ) + $ErrorActionPreference = "Stop" + + if (Get-Process -Name kube-proxy -ErrorAction SilentlyContinue) { + Write-Host "kube-proxy is already in running" + return + } + + New-DirectoryIfNotExist $LogDir + + New-KubeProxyServiceInterface + + Start-Process -FilePath $KubeProxy -ArgumentList "--proxy-mode=userspace --kubeconfig=$KubeConfig --log-dir=$LogDir --logtostderr=false --alsologtostderr --v=4" +} + +function Start-AntreaAgent { + Param( + [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea", + [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config", + [parameter(Mandatory = $false)] [string] $LogDir + ) + $ErrorActionPreference = "Stop" + + if (Get-Process -Name antrea-agent -ErrorAction SilentlyContinue) { + Write-Host "antrea-agent is already in running" + return + } + + # Try to cleanup ovsdb-server configurations if the antrea-hnsnetwork is not existing. + if (!(Get-VMswitch -Name "antrea-hnsnetwork" -SwitchType External -ErrorAction SilentlyContinue)) { + while ($true) { + & ovs-vsctl.exe --no-wait --if-exists del-br br-int + if (-Not $LASTEXITCODE) { + break + } else { + Write-Host "Waiting for ovsdb-server running" + Start-Sleep -Seconds 1 + } + } + } + while ($true) { + $service = Get-Service -Name ovs-vswitchd + if ($service.Status -eq "Running") { + break + } + if ($service.Status -eq "Stopped") { + Start-Service ovs-vswitchd + } + Write-Host "Waiting for ovs-vswitchd running" + Start-Sleep -Seconds 2 + } + + $AntreaAgent = "$AntreaHome\bin\antrea-agent.exe" + $AntreaAgentConfigPath = "$AntreaHome\etc\antrea-agent.conf" + if ($LogDir -eq "") { + $LogDir = "$AntreaHome\logs" + } + New-DirectoryIfNotExist $LogDir + [Environment]::SetEnvironmentVariable("NODE_NAME", (hostname).ToLower()) + Start-Process -FilePath $AntreaAgent -ArgumentList "--config=$AntreaAgentConfigPath --logtostderr=false --log_dir=$LogDir --alsologtostderr --log_file_max_size=100 --log_file_max_num=4" +} + +Export-ModuleMember Get-WebFileIfNotExist +Export-ModuleMember New-DirectoryIfNotExist +Export-ModuleMember Test-ConnectionWithRetry +Export-ModuleMember Install-AntreaAgent +Export-ModuleMember New-KubeProxyServiceInterface +Export-ModuleMember Start-KubeProxy +Export-ModuleMember Start-AntreaAgent diff --git a/hack/windows/Start.ps1 b/hack/windows/Start.ps1 new file mode 100644 index 00000000000..f0567d5d662 --- /dev/null +++ b/hack/windows/Start.ps1 @@ -0,0 +1,63 @@ +Param( + [parameter(Mandatory = $false, HelpMessage="Kubernetes version to use")] [string] $KubernetesVersion="v1.18.0", + [parameter(Mandatory = $false, HelpMessage="Kubernetes home path")] [string] $KubernetesHome="c:\k", + [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config", + [parameter(Mandatory = $false, HelpMessage="Antrea version to use")] [string] $AntreaVersion="latest", + [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea" +) +$ErrorActionPreference = "Stop" + +$Owner = "vmware-tanzu" +$Repo = "antrea" +if ($AntreaVersion -eq "latest") { + $AntreaReleases = (curl.exe -s "https://api.github.com/repos/$Owner/$Repo/releases" | ConvertFrom-Json) + $AntreaVersion = $AntreaReleases[0].tag_name +} + +if ($AntreaVersion -eq "") { + $ErrorMsg = ($AntreaReleases).message + Write-Host "Failed to get Antrea latest version: $ErrorMsg, exit" + exit 1 +} + +Write-Host "KubernetesVersion version: $KubernetesVersion" +Write-Host "Antrea version: $AntreaVersion" +$AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion" + +if (!(Test-Path $AntreaHome)) { + mkdir $AntreaHome +} + +$helper = "$AntreaHome\Helper.psm1" +if (!(Test-Path $helper)) +{ + curl.exe -sLo $helper "$AntreaRawUrlBase/hack/windows/Helper.psm1" +} +Import-Module $helper + +Write-Host "Checking kube-proxy and antrea-agent installation..." +Install-AntreaAgent -KubernetesVersion $KubernetesVersion -KubernetesHome $KubernetesHome -KubeConfig $KubeConfig -AntreaVersion $AntreaVersion -AntreaHome $AntreaHome + +if ($LastExitCode) { + Write-Host "Install antrea-agent failed, exit" + exit 1 +} + +Write-Host "Starting kube-proxy..." +Start-KubeProxy -KubeProxy $KubernetesHome\kube-proxy.exe -KubeConfig $KubeConfig + +$env:kubeconfig = $KubeConfig +$APIServer=$(kubectl get service kubernetes -o jsonpath='{.spec.clusterIP}') +$APIServerPort=$(kubectl get service kubernetes -o jsonpath='{.spec.ports[0].port}') +$APIServer="https://$APIServer" + ":" + $APIServerPort +$APIServer=[System.Uri]$APIServer + +Write-Host "Test connection to kubernetes API server" +$result = Test-ConnectionWithRetry $APIServer.Host $APIServer.Port 20 3 +if (!$result) { + Write-Host "Failed to connection to kubernetes API server service, exit" + exit 1 +} + +Write-Host "Starting antrea-agent..." +Start-AntreaAgent -AntreaHome $AntreaHome -KubeConfig $KubeConfig diff --git a/hack/windows/Stop.ps1 b/hack/windows/Stop.ps1 new file mode 100644 index 00000000000..2ec6b634bf9 --- /dev/null +++ b/hack/windows/Stop.ps1 @@ -0,0 +1,2 @@ +taskkill /im antrea-agent.exe /f +taskkill /im kube-proxy.exe /f diff --git a/pkg/agent/apiserver/apiserver.go b/pkg/agent/apiserver/apiserver.go index 53157ddb3cc..e4d763a6aba 100644 --- a/pkg/agent/apiserver/apiserver.go +++ b/pkg/agent/apiserver/apiserver.go @@ -89,8 +89,8 @@ func installAPIGroup(s *genericapiserver.GenericAPIServer, aq agentquerier.Agent // New creates an APIServer for running in antrea agent. func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier, bindPort int, - enableMetrics bool) (*agentAPIServer, error) { - cfg, err := newConfig(bindPort, enableMetrics) + enableMetrics bool, kubeconfig string) (*agentAPIServer, error) { + cfg, err := newConfig(bindPort, enableMetrics, kubeconfig) if err != nil { return nil, err } @@ -105,11 +105,17 @@ func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier return &agentAPIServer{GenericAPIServer: s}, nil } -func newConfig(bindPort int, enableMetrics bool) (*genericapiserver.CompletedConfig, error) { +func newConfig(bindPort int, enableMetrics bool, kubeconfig string) (*genericapiserver.CompletedConfig, error) { secureServing := genericoptions.NewSecureServingOptions().WithLoopback() authentication := genericoptions.NewDelegatingAuthenticationOptions() authorization := genericoptions.NewDelegatingAuthorizationOptions().WithAlwaysAllowPaths("/healthz") + // kubeconfig file is useful when antrea-agent isn't not running as a pod + if len(kubeconfig) > 0 { + authentication.RemoteKubeConfigFile = kubeconfig + authorization.RemoteKubeConfigFile = kubeconfig + } + // Set the PairName but leave certificate directory blank to generate in-memory by default. secureServing.ServerCert.CertDirectory = "" secureServing.ServerCert.PairName = Name