Merge pull request #143 from davidvossel/network-fixes

Network fixes

e2e/common_test.go

@@ -64,6 +64,7 @@ type tenantClusterAccess struct {
 	listener             net.Listener
 	namespace            string
 	tenantKubeconfigFile string
+	isForwarding         bool
 }
 
 func newTenantClusterAccess(namespace string, tenantKubeconfigFile string) tenantClusterAccess {
@@ -93,10 +94,18 @@ func (t *tenantClusterAccess) generateClient() (*kubernetes.Clientset, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	return kubernetes.NewForConfig(restConfig)
 }
 
+func (t *tenantClusterAccess) getLocalPort() int {
+	return t.listener.Addr().(*net.TCPAddr).Port
+}
+
 func (t *tenantClusterAccess) startForwardingTenantAPI() error {
+	if t.isForwarding {
+		return nil
+	}
 	address, err := net.ResolveIPAddr("", "127.0.0.1")
 	if err != nil {
 		return err
@@ -116,6 +125,7 @@ func (t *tenantClusterAccess) startForwardingTenantAPI() error {
 		return err
 	}
 
+	t.isForwarding = true
 	go t.waitForConnection(vmiName, t.namespace)
 
 	return nil
@@ -141,6 +151,10 @@ func (t *tenantClusterAccess) findControlPlaneVMIName() (string, error) {
 }
 
 func (t *tenantClusterAccess) stopForwardingTenantAPI() error {
+	if !t.isForwarding {
+		return nil
+	}
+	t.isForwarding = false
 	return t.listener.Close()
 }
 

e2e/create-cluster_test.go

@@ -18,6 +18,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/client-go/kubernetes"
 	kubevirtv1 "kubevirt.io/api/core/v1"
 	"kubevirt.io/client-go/kubecli"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
@@ -38,6 +39,9 @@ var _ = Describe("CreateCluster", func() {
 	var manifestsFile string
 	var tenantKubeconfigFile string
 	var namespace string
+	var tenantAccessor tenantClusterAccess
+
+	calicoManifestsUrl := "https://docs.projectcalico.org/v3.21/manifests/calico.yaml"
 
 	BeforeEach(func() {
 		var err error
@@ -68,6 +72,7 @@ var _ = Describe("CreateCluster", func() {
 			},
 		}
 
+		tenantAccessor = newTenantClusterAccess(namespace, tenantKubeconfigFile)
 		err = k8sclient.Create(context.Background(), ns)
 		Expect(err).ToNot(HaveOccurred())
 	})
@@ -88,6 +93,8 @@ var _ = Describe("CreateCluster", func() {
 
 		_ = os.RemoveAll(tmpDir)
 
+		_ = tenantAccessor.stopForwardingTenantAPI()
+
 		By("removing cluster")
 		cluster := &clusterv1.Cluster{
 			ObjectMeta: metav1.ObjectMeta{
@@ -235,18 +242,43 @@ var _ = Describe("CreateCluster", func() {
 		}, 5*time.Minute, 5*time.Second).Should(Succeed(), "waiting for expected readiness.")
 	}
 
-	waitForTenantAccess := func(numExpectedNodes int) {
+	waitForTenantPods := func() {
+
 		By(fmt.Sprintf("Perform Port Forward using controlplane vmi in namespace %s", namespace))
-		t := newTenantClusterAccess(namespace, tenantKubeconfigFile)
-		err := t.startForwardingTenantAPI()
+		err := tenantAccessor.startForwardingTenantAPI()
 		Expect(err).ToNot(HaveOccurred())
-		defer func() {
-			err := t.stopForwardingTenantAPI()
+
+		By("Create client to access the tenant cluster")
+		clientSet, err := tenantAccessor.generateClient()
+		Expect(err).ToNot(HaveOccurred())
+
+		Eventually(func() error {
+			podList, err := clientSet.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{})
 			Expect(err).ToNot(HaveOccurred())
-		}()
+
+			offlinePodList := []string{}
+			for _, pod := range podList.Items {
+				if pod.Status.Phase != corev1.PodRunning {
+					offlinePodList = append(offlinePodList, pod.Name)
+				}
+			}
+
+			if len(offlinePodList) > 0 {
+
+				return fmt.Errorf("Waiting on tenant pods [%v] to reach a Running phase", offlinePodList)
+			}
+			return nil
+		}, 8*time.Minute, 5*time.Second).Should(Succeed(), "waiting for pods to hit Running phase.")
+
+	}
+
+	waitForTenantAccess := func(numExpectedNodes int) *kubernetes.Clientset {
+		By(fmt.Sprintf("Perform Port Forward using controlplane vmi in namespace %s", namespace))
+		err := tenantAccessor.startForwardingTenantAPI()
+		Expect(err).ToNot(HaveOccurred())
 
 		By("Create client to access the tenant cluster")
-		clientSet, err := t.generateClient()
+		clientSet, err := tenantAccessor.generateClient()
 		Expect(err).ToNot(HaveOccurred())
 
 		Eventually(func() error {
@@ -259,6 +291,52 @@ var _ = Describe("CreateCluster", func() {
 
 			return nil
 		}, 5*time.Minute, 5*time.Second).Should(Succeed(), "waiting for expected readiness.")
+
+		return clientSet
+	}
+
+	waitForNodeReadiness := func() *kubernetes.Clientset {
+		By(fmt.Sprintf("Perform Port Forward using controlplane vmi in namespace %s", namespace))
+		err := tenantAccessor.startForwardingTenantAPI()
+		Expect(err).ToNot(HaveOccurred())
+
+		By("Create client to access the tenant cluster")
+		clientSet, err := tenantAccessor.generateClient()
+		Expect(err).ToNot(HaveOccurred())
+
+		Eventually(func() error {
+
+			nodeList, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range nodeList.Items {
+
+				ready := false
+				networkAvailable := false
+				for _, cond := range node.Status.Conditions {
+					if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue {
+						ready = true
+					} else if cond.Type == corev1.NodeNetworkUnavailable && cond.Status == corev1.ConditionFalse {
+						networkAvailable = true
+					}
+				}
+
+				if !ready {
+					return fmt.Errorf("waiting on node %s to become ready", node.Name)
+				} else if !networkAvailable {
+					return fmt.Errorf("waiting on node %s to have network availablity", node.Name)
+				}
+			}
+
+			return nil
+		}, 8*time.Minute, 5*time.Second).Should(Succeed(), "ensure healthy nodes.")
+
+		return clientSet
+	}
+
+	installCalicoCNI := func() {
+		cmd := exec.Command(KubectlPath, "--kubeconfig", tenantKubeconfigFile, "--insecure-skip-tls-verify", "--server", fmt.Sprintf("https://localhost:%d", tenantAccessor.getLocalPort()), "apply", "-f", calicoManifestsUrl)
+		RunCmd(cmd)
 	}
 
 	waitForNodeUpdate := func() {
@@ -455,6 +533,15 @@ var _ = Describe("CreateCluster", func() {
 
 		By("Waiting for getting access to the tenant cluster")
 		waitForTenantAccess(2)
+
+		By("posting calico CNI manifests to the guest cluster and waiting for network")
+		installCalicoCNI()
+
+		By("Waiting for node readiness")
+		waitForNodeReadiness()
+
+		By("waiting all tenant Pods to be Ready")
+		waitForTenantPods()
 	})
 
 	It("should remediate a running VMI marked as being in a terminal state", Label("ephemeralVMs"), func() {
@@ -515,6 +602,15 @@ var _ = Describe("CreateCluster", func() {
 		By("Waiting for getting access to the tenant cluster")
 		waitForTenantAccess(2)
 
+		By("posting calico CNI manifests to the guest cluster and waiting for network")
+		installCalicoCNI()
+
+		By("Waiting for node readiness")
+		waitForNodeReadiness()
+
+		By("waiting all tenant Pods to be Ready")
+		waitForTenantPods()
+
 	})
 
 	It("should remediate failed unrecoverable VMI ", Label("ephemeralVMs"), func() {
@@ -683,5 +779,14 @@ var _ = Describe("CreateCluster", func() {
 
 		By("Waiting for getting access to the tenant cluster")
 		waitForTenantAccess(2)
+
+		By("posting calico CNI manifests to the guest cluster and waiting for network")
+		installCalicoCNI()
+
+		By("Waiting for node readiness")
+		waitForNodeReadiness()
+
+		By("waiting all tenant Pods to be Ready")
+		waitForTenantPods()
 	})
 })

kubevirtci

@@ -2,8 +2,9 @@
 
 set -e
 
-export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.21}
-export KUBEVIRTCI_TAG=${KUBEVIRTCI_TAG:-2110251848-8198e9c}
+export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.23}
+export CAPK_GUEST_K8S_VERSION=${CAPK_GUEST_K8S_VERSION:-v1.21.0}
+export KUBEVIRTCI_TAG=${KUBEVIRTCI_TAG:-2205231118-f12b50e}
 export KUBECONFIG=$(cluster-up/cluster-up/kubeconfig.sh)
 export KUBEVIRT_DEPLOY_PROMETHEUS=false
 export KUBEVIRT_DEPLOY_CDI=false
@@ -14,6 +15,7 @@ export KUBEVIRT_STORAGE="rook-ceph-default"
 export METALLB_VERSION="v0.12.1"
 export CAPK_RELEASE_VERSION="v0.1.0-rc.0"
 export CLUSTERCTL_VERSION="v1.0.0"
+export CALICO_VERSION="v3.21"
 
 _default_bin_path=./hack/tools/bin
 _default_tmp_path=./hack/tools/bin/tmp
@@ -52,6 +54,7 @@ function kubevirtci::usage() {
 
 	  install-capk                      Installs capk from published release manifests
 	  install-metallb                   Installs metallb into the infra cluster
+	  install-calico                    Installs calico cni into tenant cluster
 	  curl-lb <lb name> [lb namespace]  Curls lb service within infra cluster
 
 	  ssh-infra <node name>             SSH into one of the infra nodes (like node01)
@@ -167,7 +170,7 @@ function kubevirtci::create_cluster() {
 
 	echo "Using cluster template $template"
 
-	$CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version v1.21.0 --control-plane-machine-count=1 --worker-machine-count=1 --from $template | ${_kubectl} apply -f -
+	$CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version ${CAPK_GUEST_K8S_VERSION} --control-plane-machine-count=1 --worker-machine-count=1 --from $template | ${_kubectl} apply -f -
 }
 
 function kubevirtci::create_external_cluster() {
@@ -177,7 +180,7 @@ function kubevirtci::create_external_cluster() {
 
 	${_kubectl} delete secret external-infra-kubeconfig -n capk-system --ignore-not-found
 	${_kubectl} create secret generic external-infra-kubeconfig -n capk-system --from-file=kubeconfig=kubeconfig-e2e --from-literal=namespace=${TENANT_CLUSTER_NAMESPACE}
-	$CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version v1.21.0 --control-plane-machine-count=1 --worker-machine-count=1 --from templates/cluster-template-ext-infra.yaml | ${_kubectl} apply -f -
+	$CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version ${CAPK_GUEST_K8S_VERSION} --control-plane-machine-count=1 --worker-machine-count=1 --from templates/cluster-template-ext-infra.yaml | ${_kubectl} apply -f -
 }
 
 function kubevirtci::create_tenant_namespace {
@@ -198,6 +201,11 @@ function kubevirtci::install_capk_release {
 	echo "capk release $CAPK_RELEASE_VERSION installed!"
 }
 
+
+function kubevirtci::install_calico {
+	kubevirtci::kubectl_tenant apply -f https://docs.projectcalico.org/${CALICO_VERSION}/manifests/calico.yaml
+}
+
 function kubevirtci::install_metallb {
 	${_kubectl} apply -f https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/manifests/namespace.yaml
 	${_kubectl} apply -f https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/manifests/metallb.yaml
@@ -324,6 +332,9 @@ case ${_action} in
 "install-metallb")
 	kubevirtci::install_metallb
 	;;
+"install-calico")
+	kubevirtci::install_calico
+	;;
 "curl-lb")
 	kubevirtci::curl_lb "$@"
 	;;

templates/cluster-template-ext-infra.yaml

@@ -8,10 +8,10 @@ spec:
   clusterNetwork:
     pods:
       cidrBlocks:
-        - 10.244.0.0/16
+        - 10.243.0.0/16
     services:
       cidrBlocks:
-        - 10.96.0.0/12
+        - 10.95.0.0/16
   infrastructureRef:
     apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
     kind: KubevirtCluster
@@ -79,6 +79,10 @@ spec:
   kubeadmConfigSpec:
     clusterConfiguration:
       imageRepository: ${IMAGE_REPO}
+      networking:
+        dnsDomain: "${CLUSTER_NAME}.${NAMESPACE}.local"
+        podSubnet: 10.243.0.0/16
+        serviceSubnet: 10.95.0.0/16
     initConfiguration:
       nodeRegistration:
         criSocket: "${CRI_PATH}"

templates/cluster-template-persistent-storage.yaml

@@ -8,10 +8,10 @@ spec:
   clusterNetwork:
     pods:
       cidrBlocks:
-        - 10.244.0.0/16
+        - 10.243.0.0/16
     services:
       cidrBlocks:
-        - 10.96.0.0/12
+        - 10.95.0.0/16
   infrastructureRef:
     apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
     kind: KubevirtCluster
@@ -89,6 +89,10 @@ spec:
   kubeadmConfigSpec:
     clusterConfiguration:
       imageRepository: ${IMAGE_REPO}
+      networking:
+        dnsDomain: "${CLUSTER_NAME}.${NAMESPACE}.local"
+        podSubnet: 10.243.0.0/16
+        serviceSubnet: 10.95.0.0/16
     initConfiguration:
       nodeRegistration:
         criSocket: "${CRI_PATH}"

templates/cluster-template.yaml

@@ -8,10 +8,10 @@ spec:
   clusterNetwork:
     pods:
       cidrBlocks:
-        - 10.244.0.0/16
+        - 10.243.0.0/16
     services:
       cidrBlocks:
-        - 10.96.0.0/12
+        - 10.95.0.0/16
   infrastructureRef:
     apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
     kind: KubevirtCluster
@@ -79,6 +79,10 @@ spec:
   kubeadmConfigSpec:
     clusterConfiguration:
       imageRepository: ${IMAGE_REPO}
+      networking:
+        dnsDomain: "${CLUSTER_NAME}.${NAMESPACE}.local"
+        podSubnet: 10.243.0.0/16
+        serviceSubnet: 10.95.0.0/16
     initConfiguration:
       nodeRegistration:
         criSocket: "${CRI_PATH}"