From a6e458474331f680e908e7a265a47c2982735bd2 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 1 Jun 2022 10:44:34 -0400 Subject: [PATCH 1/3] add install calico to kubevirtci and update cluster-up git hash Signed-off-by: David Vossel --- kubevirtci | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/kubevirtci b/kubevirtci index f00cb883..47773979 100755 --- a/kubevirtci +++ b/kubevirtci @@ -2,8 +2,9 @@ set -e -export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.21} -export KUBEVIRTCI_TAG=${KUBEVIRTCI_TAG:-2110251848-8198e9c} +export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.23} +export CAPK_GUEST_K8S_VERSION=${CAPK_GUEST_K8S_VERSION:-v1.21.0} +export KUBEVIRTCI_TAG=${KUBEVIRTCI_TAG:-2205231118-f12b50e} export KUBECONFIG=$(cluster-up/cluster-up/kubeconfig.sh) export KUBEVIRT_DEPLOY_PROMETHEUS=false export KUBEVIRT_DEPLOY_CDI=false @@ -14,6 +15,7 @@ export KUBEVIRT_STORAGE="rook-ceph-default" export METALLB_VERSION="v0.12.1" export CAPK_RELEASE_VERSION="v0.1.0-rc.0" export CLUSTERCTL_VERSION="v1.0.0" +export CALICO_VERSION="v3.21" _default_bin_path=./hack/tools/bin _default_tmp_path=./hack/tools/bin/tmp @@ -52,6 +54,7 @@ function kubevirtci::usage() { install-capk Installs capk from published release manifests install-metallb Installs metallb into the infra cluster + install-calico Installs calico cni into tenant cluster curl-lb [lb namespace] Curls lb service within infra cluster ssh-infra SSH into one of the infra nodes (like node01) @@ -167,7 +170,7 @@ function kubevirtci::create_cluster() { echo "Using cluster template $template" - $CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version v1.21.0 --control-plane-machine-count=1 --worker-machine-count=1 --from $template | ${_kubectl} apply -f - + $CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version ${CAPK_GUEST_K8S_VERSION} --control-plane-machine-count=1 --worker-machine-count=1 --from $template | ${_kubectl} apply -f - } function kubevirtci::create_external_cluster() { @@ -177,7 +180,7 @@ function kubevirtci::create_external_cluster() { ${_kubectl} delete secret external-infra-kubeconfig -n capk-system --ignore-not-found ${_kubectl} create secret generic external-infra-kubeconfig -n capk-system --from-file=kubeconfig=kubeconfig-e2e --from-literal=namespace=${TENANT_CLUSTER_NAMESPACE} - $CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version v1.21.0 --control-plane-machine-count=1 --worker-machine-count=1 --from templates/cluster-template-ext-infra.yaml | ${_kubectl} apply -f - + $CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version ${CAPK_GUEST_K8S_VERSION} --control-plane-machine-count=1 --worker-machine-count=1 --from templates/cluster-template-ext-infra.yaml | ${_kubectl} apply -f - } function kubevirtci::create_tenant_namespace { @@ -198,6 +201,11 @@ function kubevirtci::install_capk_release { echo "capk release $CAPK_RELEASE_VERSION installed!" } + +function kubevirtci::install_calico { + kubevirtci::kubectl_tenant apply -f https://docs.projectcalico.org/${CALICO_VERSION}/manifests/calico.yaml +} + function kubevirtci::install_metallb { ${_kubectl} apply -f https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/manifests/namespace.yaml ${_kubectl} apply -f https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/manifests/metallb.yaml @@ -324,6 +332,9 @@ case ${_action} in "install-metallb") kubevirtci::install_metallb ;; +"install-calico") + kubevirtci::install_calico + ;; "curl-lb") kubevirtci::curl_lb "$@" ;; From 7880b04a3adab6892239e20381f8756542ed43f4 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 1 Jun 2022 10:47:47 -0400 Subject: [PATCH 2/3] offset tenant network settings from default infra kubeadm settings The domain and cidr ranges of the tenant cluster need to be offset from the default kubeadm settings or else they will likely conflict and overlap with the infra network's settings Signed-off-by: David Vossel --- templates/cluster-template-ext-infra.yaml | 8 ++++++-- templates/cluster-template-persistent-storage.yaml | 8 ++++++-- templates/cluster-template.yaml | 8 ++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/templates/cluster-template-ext-infra.yaml b/templates/cluster-template-ext-infra.yaml index 029a74b5..e0d417c1 100644 --- a/templates/cluster-template-ext-infra.yaml +++ b/templates/cluster-template-ext-infra.yaml @@ -8,10 +8,10 @@ spec: clusterNetwork: pods: cidrBlocks: - - 10.244.0.0/16 + - 10.243.0.0/16 services: cidrBlocks: - - 10.96.0.0/12 + - 10.95.0.0/16 infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: KubevirtCluster @@ -79,6 +79,10 @@ spec: kubeadmConfigSpec: clusterConfiguration: imageRepository: ${IMAGE_REPO} + networking: + dnsDomain: "${CLUSTER_NAME}.${NAMESPACE}.local" + podSubnet: 10.243.0.0/16 + serviceSubnet: 10.95.0.0/16 initConfiguration: nodeRegistration: criSocket: "${CRI_PATH}" diff --git a/templates/cluster-template-persistent-storage.yaml b/templates/cluster-template-persistent-storage.yaml index 8889085f..c83a1636 100644 --- a/templates/cluster-template-persistent-storage.yaml +++ b/templates/cluster-template-persistent-storage.yaml @@ -8,10 +8,10 @@ spec: clusterNetwork: pods: cidrBlocks: - - 10.244.0.0/16 + - 10.243.0.0/16 services: cidrBlocks: - - 10.96.0.0/12 + - 10.95.0.0/16 infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: KubevirtCluster @@ -89,6 +89,10 @@ spec: kubeadmConfigSpec: clusterConfiguration: imageRepository: ${IMAGE_REPO} + networking: + dnsDomain: "${CLUSTER_NAME}.${NAMESPACE}.local" + podSubnet: 10.243.0.0/16 + serviceSubnet: 10.95.0.0/16 initConfiguration: nodeRegistration: criSocket: "${CRI_PATH}" diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 782d1bd3..a56775d5 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -8,10 +8,10 @@ spec: clusterNetwork: pods: cidrBlocks: - - 10.244.0.0/16 + - 10.243.0.0/16 services: cidrBlocks: - - 10.96.0.0/12 + - 10.95.0.0/16 infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: KubevirtCluster @@ -79,6 +79,10 @@ spec: kubeadmConfigSpec: clusterConfiguration: imageRepository: ${IMAGE_REPO} + networking: + dnsDomain: "${CLUSTER_NAME}.${NAMESPACE}.local" + podSubnet: 10.243.0.0/16 + serviceSubnet: 10.95.0.0/16 initConfiguration: nodeRegistration: criSocket: "${CRI_PATH}" From 7c2ee63569d039bbff60d4e5a60a717819ac7c1f Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 1 Jun 2022 14:13:42 -0400 Subject: [PATCH 3/3] Install cni and verify pod/node status in e2e tests Signed-off-by: David Vossel --- e2e/common_test.go | 14 +++++ e2e/create-cluster_test.go | 119 ++++++++++++++++++++++++++++++++++--- 2 files changed, 126 insertions(+), 7 deletions(-) diff --git a/e2e/common_test.go b/e2e/common_test.go index 35208975..e61719e1 100644 --- a/e2e/common_test.go +++ b/e2e/common_test.go @@ -64,6 +64,7 @@ type tenantClusterAccess struct { listener net.Listener namespace string tenantKubeconfigFile string + isForwarding bool } func newTenantClusterAccess(namespace string, tenantKubeconfigFile string) tenantClusterAccess { @@ -93,10 +94,18 @@ func (t *tenantClusterAccess) generateClient() (*kubernetes.Clientset, error) { if err != nil { return nil, err } + return kubernetes.NewForConfig(restConfig) } +func (t *tenantClusterAccess) getLocalPort() int { + return t.listener.Addr().(*net.TCPAddr).Port +} + func (t *tenantClusterAccess) startForwardingTenantAPI() error { + if t.isForwarding { + return nil + } address, err := net.ResolveIPAddr("", "127.0.0.1") if err != nil { return err @@ -116,6 +125,7 @@ func (t *tenantClusterAccess) startForwardingTenantAPI() error { return err } + t.isForwarding = true go t.waitForConnection(vmiName, t.namespace) return nil @@ -141,6 +151,10 @@ func (t *tenantClusterAccess) findControlPlaneVMIName() (string, error) { } func (t *tenantClusterAccess) stopForwardingTenantAPI() error { + if !t.isForwarding { + return nil + } + t.isForwarding = false return t.listener.Close() } diff --git a/e2e/create-cluster_test.go b/e2e/create-cluster_test.go index f1815146..70d7cd6e 100644 --- a/e2e/create-cluster_test.go +++ b/e2e/create-cluster_test.go @@ -18,6 +18,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/kubernetes" kubevirtv1 "kubevirt.io/api/core/v1" "kubevirt.io/client-go/kubecli" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -38,6 +39,9 @@ var _ = Describe("CreateCluster", func() { var manifestsFile string var tenantKubeconfigFile string var namespace string + var tenantAccessor tenantClusterAccess + + calicoManifestsUrl := "https://docs.projectcalico.org/v3.21/manifests/calico.yaml" BeforeEach(func() { var err error @@ -68,6 +72,7 @@ var _ = Describe("CreateCluster", func() { }, } + tenantAccessor = newTenantClusterAccess(namespace, tenantKubeconfigFile) err = k8sclient.Create(context.Background(), ns) Expect(err).ToNot(HaveOccurred()) }) @@ -88,6 +93,8 @@ var _ = Describe("CreateCluster", func() { _ = os.RemoveAll(tmpDir) + _ = tenantAccessor.stopForwardingTenantAPI() + By("removing cluster") cluster := &clusterv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ @@ -235,18 +242,43 @@ var _ = Describe("CreateCluster", func() { }, 5*time.Minute, 5*time.Second).Should(Succeed(), "waiting for expected readiness.") } - waitForTenantAccess := func(numExpectedNodes int) { + waitForTenantPods := func() { + By(fmt.Sprintf("Perform Port Forward using controlplane vmi in namespace %s", namespace)) - t := newTenantClusterAccess(namespace, tenantKubeconfigFile) - err := t.startForwardingTenantAPI() + err := tenantAccessor.startForwardingTenantAPI() Expect(err).ToNot(HaveOccurred()) - defer func() { - err := t.stopForwardingTenantAPI() + + By("Create client to access the tenant cluster") + clientSet, err := tenantAccessor.generateClient() + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() error { + podList, err := clientSet.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{}) Expect(err).ToNot(HaveOccurred()) - }() + + offlinePodList := []string{} + for _, pod := range podList.Items { + if pod.Status.Phase != corev1.PodRunning { + offlinePodList = append(offlinePodList, pod.Name) + } + } + + if len(offlinePodList) > 0 { + + return fmt.Errorf("Waiting on tenant pods [%v] to reach a Running phase", offlinePodList) + } + return nil + }, 8*time.Minute, 5*time.Second).Should(Succeed(), "waiting for pods to hit Running phase.") + + } + + waitForTenantAccess := func(numExpectedNodes int) *kubernetes.Clientset { + By(fmt.Sprintf("Perform Port Forward using controlplane vmi in namespace %s", namespace)) + err := tenantAccessor.startForwardingTenantAPI() + Expect(err).ToNot(HaveOccurred()) By("Create client to access the tenant cluster") - clientSet, err := t.generateClient() + clientSet, err := tenantAccessor.generateClient() Expect(err).ToNot(HaveOccurred()) Eventually(func() error { @@ -259,6 +291,52 @@ var _ = Describe("CreateCluster", func() { return nil }, 5*time.Minute, 5*time.Second).Should(Succeed(), "waiting for expected readiness.") + + return clientSet + } + + waitForNodeReadiness := func() *kubernetes.Clientset { + By(fmt.Sprintf("Perform Port Forward using controlplane vmi in namespace %s", namespace)) + err := tenantAccessor.startForwardingTenantAPI() + Expect(err).ToNot(HaveOccurred()) + + By("Create client to access the tenant cluster") + clientSet, err := tenantAccessor.generateClient() + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() error { + + nodeList, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + Expect(err).ToNot(HaveOccurred()) + + for _, node := range nodeList.Items { + + ready := false + networkAvailable := false + for _, cond := range node.Status.Conditions { + if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue { + ready = true + } else if cond.Type == corev1.NodeNetworkUnavailable && cond.Status == corev1.ConditionFalse { + networkAvailable = true + } + } + + if !ready { + return fmt.Errorf("waiting on node %s to become ready", node.Name) + } else if !networkAvailable { + return fmt.Errorf("waiting on node %s to have network availablity", node.Name) + } + } + + return nil + }, 8*time.Minute, 5*time.Second).Should(Succeed(), "ensure healthy nodes.") + + return clientSet + } + + installCalicoCNI := func() { + cmd := exec.Command(KubectlPath, "--kubeconfig", tenantKubeconfigFile, "--insecure-skip-tls-verify", "--server", fmt.Sprintf("https://localhost:%d", tenantAccessor.getLocalPort()), "apply", "-f", calicoManifestsUrl) + RunCmd(cmd) } waitForNodeUpdate := func() { @@ -455,6 +533,15 @@ var _ = Describe("CreateCluster", func() { By("Waiting for getting access to the tenant cluster") waitForTenantAccess(2) + + By("posting calico CNI manifests to the guest cluster and waiting for network") + installCalicoCNI() + + By("Waiting for node readiness") + waitForNodeReadiness() + + By("waiting all tenant Pods to be Ready") + waitForTenantPods() }) It("should remediate a running VMI marked as being in a terminal state", Label("ephemeralVMs"), func() { @@ -515,6 +602,15 @@ var _ = Describe("CreateCluster", func() { By("Waiting for getting access to the tenant cluster") waitForTenantAccess(2) + By("posting calico CNI manifests to the guest cluster and waiting for network") + installCalicoCNI() + + By("Waiting for node readiness") + waitForNodeReadiness() + + By("waiting all tenant Pods to be Ready") + waitForTenantPods() + }) It("should remediate failed unrecoverable VMI ", Label("ephemeralVMs"), func() { @@ -683,5 +779,14 @@ var _ = Describe("CreateCluster", func() { By("Waiting for getting access to the tenant cluster") waitForTenantAccess(2) + + By("posting calico CNI manifests to the guest cluster and waiting for network") + installCalicoCNI() + + By("Waiting for node readiness") + waitForNodeReadiness() + + By("waiting all tenant Pods to be Ready") + waitForTenantPods() }) })