diff --git a/.circleci/config.yml b/.circleci/config.yml index d7c918b9c..2efed2850 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -159,6 +159,9 @@ e2e: &e2e elif [[ ${CIRCLE_JOB} = e2e_weave ]]; then export CNI_PLUGIN=weave echo >&2 "*** Using Weave CNI" + elif [[ ${CIRCLE_JOB} = e2e_multi_cni ]]; then + export MULTI_CNI=1 + echo >&2 "*** Using multiple CNIs (flannel + calico)" fi SKIP_SNAPSHOT=1 \ NONINTERACTIVE=1 \ @@ -184,7 +187,12 @@ e2e: &e2e command: | build/portforward.sh 8080& mkdir -p ~/junit - _output/virtlet-e2e-tests -test.v -ginkgo.skip="\[Heavy\]" -ginkgo.skip="\[Disruptive\]" -junitOutput ~/junit/junit.xml -include-unsafe-tests=true + skip="-ginkgo.skip=\[Heavy\]|\[MultiCNI\]|\[Disruptive\]" + if [[ ${CIRCLE_JOB} = e2e_multi_cni ]]; then + # per-node config test requires an additional worker node + skip="${skip}|Per-node configuration" + fi + _output/virtlet-e2e-tests -test.v "${skip}" -junitOutput ~/junit/junit.xml -include-unsafe-tests=true - store_test_results: path: ~/junit @@ -348,6 +356,9 @@ jobs: e2e_weave: <<: *e2e + e2e_multi_cni: + <<: *e2e + push_branch: <<: *push_images @@ -439,6 +450,15 @@ workflows: tags: only: - /^v[0-9].*/ + - e2e_multi_cni: + requires: + - build + filters: + branches: + only: /^master$|^.*-net$/ + tags: + only: + - /^v[0-9].*/ - push_branch: requires: - build @@ -453,6 +473,7 @@ workflows: - e2e_calico - e2e_flannel - e2e_weave + - e2e_multi_cni - integration filters: branches: diff --git a/deploy/demo.sh b/deploy/demo.sh index 1227c569e..58d652532 100755 --- a/deploy/demo.sh +++ b/deploy/demo.sh @@ -18,6 +18,7 @@ VIRTLET_DEMO_BRANCH="${VIRTLET_DEMO_BRANCH:-}" VIRTLET_ON_MASTER="${VIRTLET_ON_MASTER:-}" VIRTLET_MULTI_NODE="${VIRTLET_MULTI_NODE:-}" IMAGE_REGEXP_TRANSLATION="${IMAGE_REGEXP_TRANSLATION:-1}" +MULTI_CNI="${MULTI_CNI:-}" # Convenience setting for local testing: # BASE_LOCATION="${HOME}/work/kubernetes/src/github.com/Mirantis/virtlet" cirros_key="demo-cirros-private-key" @@ -135,6 +136,25 @@ function demo::start-dind-cluster { "./${dind_script}" up } +function demo::jq-patch { + local node="${1}" + local expr="${2}" + local filename="${3}" + docker exec "${node}" \ + bash -c "jq '${expr}' '${filename}' >/tmp/jqpatch.tmp && mv /tmp/jqpatch.tmp '${filename}'" +} + +function demo::install-cni-genie { + "${kubectl}" apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml + demo::wait-for "Calico etcd" demo::pods-ready k8s-app=calico-etcd + demo::wait-for "Calico node" demo::pods-ready k8s-app=calico-node + "${kubectl}" apply -f https://raw.githubusercontent.com/Mirantis/CNI-Genie/mymaster/conf/1.8/genie.yaml + demo::wait-for "CNI Genie" demo::pods-ready k8s-app=genie + demo::jq-patch kube-node-1 '.cniVersion="0.3.0"|.default_plugin="calico,flannel"' /etc/cni/net.d/00-genie.conf + demo::jq-patch kube-node-1 '.cniVersion="0.3.0"' /etc/cni/net.d/10-calico.conf + demo::jq-patch kube-node-1 '.cniVersion="0.3.0"' /etc/cni/net.d/10-flannel.conflist +} + function demo::install-cri-proxy { local virtlet_node="${1}" demo::step "Installing CRI proxy package on ${virtlet_node} container" @@ -378,7 +398,14 @@ EOF fi demo::get-dind-cluster +if [[ ${MULTI_CNI} ]]; then + export NUM_NODES=1 + export CNI_PLUGIN=flannel +fi demo::start-dind-cluster +if [[ ${MULTI_CNI} ]]; then + demo::install-cni-genie +fi for virtlet_node in "${virtlet_nodes[@]}"; do demo::fix-mounts "${virtlet_node}" demo::install-cri-proxy "${virtlet_node}" diff --git a/tests/e2e/basic_test.go b/tests/e2e/basic_test.go index 317a0b7d4..05510567f 100644 --- a/tests/e2e/basic_test.go +++ b/tests/e2e/basic_test.go @@ -52,41 +52,9 @@ var _ = Describe("Virtlet [Basic cirros tests]", func() { var ssh framework.Executor scheduleWaitSSH(&vm, &ssh) - It("Should have default route [Conformance]", func() { - Expect(framework.RunSimple(ssh, "ip r")).To(SatisfyAll( - ContainSubstring("default via"), - ContainSubstring("src "+vmPod.Pod.Status.PodIP), - )) - }) - - It("Should have internet connectivity [Conformance]", func(done Done) { - defer close(done) - Expect(framework.RunSimple(ssh, "ping -c1 8.8.8.8")).To(MatchRegexp( - "1 .*transmitted, 1 .*received, 0% .*loss")) - }, 5) - - Context("With nginx server", func() { - var nginxPod *framework.PodInterface - - BeforeAll(func() { - p, err := controller.RunPod("nginx", "nginx", nil, time.Minute*4, 80) - Expect(err).NotTo(HaveOccurred()) - Expect(p).NotTo(BeNil()) - nginxPod = p - }) - - AfterAll(func() { - Expect(nginxPod.Delete()).To(Succeed()) - }) - - It("Should be able to access another k8s endpoint [Conformance]", func(done Done) { - defer close(done) - cmd := fmt.Sprintf("curl -s --connect-timeout 5 http://nginx.%s.svc.cluster.local", controller.Namespace()) - Eventually(func() (string, error) { - return framework.RunSimple(ssh, cmd) - }, 60).Should(ContainSubstring("Thank you for using nginx.")) - }, 60*5) - }) + itShouldHaveNetworkConnectivity( + func() *framework.PodInterface { return vmPod }, + func() framework.Executor { return ssh }) It("Should have hostname equal to the pod name [Conformance]", func() { Expect(framework.RunSimple(ssh, "hostname")).To(Equal(vmPod.Pod.Name)) @@ -216,3 +184,41 @@ var _ = Describe("Virtlet [Disruptive]", func() { Expect(vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, nil)).To(Succeed()) }) }) + +func itShouldHaveNetworkConnectivity(podIface func() *framework.PodInterface, ssh func() framework.Executor) { + It("Should have default route [Conformance]", func() { + Expect(framework.RunSimple(ssh(), "ip r")).To(SatisfyAll( + ContainSubstring("default via"), + ContainSubstring("src "+podIface().Pod.Status.PodIP), + )) + }) + + It("Should have internet connectivity [Conformance]", func(done Done) { + defer close(done) + Expect(framework.RunSimple(ssh(), "ping -c1 8.8.8.8")).To(MatchRegexp( + "1 .*transmitted, 1 .*received, 0% .*loss")) + }, 5) + + Context("With nginx server", func() { + var nginxPod *framework.PodInterface + + BeforeAll(func() { + p, err := controller.RunPod("nginx", "nginx", nil, time.Minute*4, 80) + Expect(err).NotTo(HaveOccurred()) + Expect(p).NotTo(BeNil()) + nginxPod = p + }) + + AfterAll(func() { + Expect(nginxPod.Delete()).To(Succeed()) + }) + + It("Should be able to access another k8s endpoint [Conformance]", func(done Done) { + defer close(done) + cmd := fmt.Sprintf("curl -s --connect-timeout 5 http://nginx.%s.svc.cluster.local", controller.Namespace()) + Eventually(func() (string, error) { + return framework.RunSimple(ssh(), cmd) + }, 60).Should(ContainSubstring("Thank you for using nginx.")) + }, 60*5) + }) +} diff --git a/tests/e2e/framework/vm_interface.go b/tests/e2e/framework/vm_interface.go index 4338f9b20..da6e60a2e 100644 --- a/tests/e2e/framework/vm_interface.go +++ b/tests/e2e/framework/vm_interface.go @@ -51,6 +51,7 @@ type VMOptions struct { UserDataScript string UserDataSource string NodeName string + MultiCNI string } func newVMInterface(controller *Controller, name string) *VMInterface { @@ -154,6 +155,9 @@ func (vmi *VMInterface) buildVMPod(options VMOptions) *v1.Pod { if options.VCPUCount > 0 { annotations["VirtletVCPUCount"] = strconv.Itoa(options.VCPUCount) } + if options.MultiCNI != "" { + annotations["cni"] = options.MultiCNI + } limits := v1.ResourceList{} for k, v := range options.Limits { diff --git a/tests/e2e/multi_cni_test.go b/tests/e2e/multi_cni_test.go new file mode 100644 index 000000000..9d269fd63 --- /dev/null +++ b/tests/e2e/multi_cni_test.go @@ -0,0 +1,149 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "time" + + . "github.com/onsi/gomega" + + "github.com/Mirantis/virtlet/tests/e2e/framework" + . "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext" +) + +const ( + ensureEth1UpCmd = "export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin';" + + "(ip a show dev eth1 | grep -qw inet) || " + + "( (echo -e 'iface eth1 inet dhcp' | " + + "sudo tee -a /etc/network/interfaces) && sudo /sbin/ifup eth1)" + getLinkIpCmd = "export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin';" + + "ip a show dev eth%d | grep -w inet | " + + "sed 's@/.*@@' | awk '{ print $2; }'" + netcatListenCmd = "echo iwaslistening | nc -l -p 12345 | grep isentthis" + netcatSendCmd = "echo isentthis | nc -w5 %s 12345 | grep iwaslistening" +) + +var _ = Describe("VMs with multiple CNIs using CNI Genie [MultiCNI]", func() { + describeMultiCNI("With 'cni' annotation", true) + describeMultiCNI("Without 'cni' annotation", false) +}) + +func describeMultiCNI(what string, addCNIAnnotation bool) { + Context(what, func() { + var ( + vms [2]*multiCNIVM + ) + BeforeAll(func() { + for i := 0; i < 2; i++ { + vms[i] = makeMultiCNIVM(fmt.Sprintf("vm%d", i), addCNIAnnotation) + vms[i].ensureIPOnSecondEth() + vms[i].retrieveIPs() + } + }) + + AfterAll(func() { + for _, vm := range vms { + vm.teardown() + } + }) + + It("Should have connectivity between them on all the CNI-provided interfaces inside VMs", func() { + vms[0].ping(vms[1].ips[0]) + vms[0].ping(vms[1].ips[1]) + vms[1].ping(vms[0].ips[0]) + vms[1].ping(vms[0].ips[1]) + errCh := make(chan error) + go func() { + errCh <- vms[0].netcatListen(vms[0].ips[0]) + }() + vms[1].netcatConnect(vms[0].ips[0]) + Expect(<-errCh).To(Succeed()) + go func() { + errCh <- vms[0].netcatListen(vms[0].ips[1]) + }() + vms[1].netcatConnect(vms[0].ips[1]) + Expect(<-errCh).To(Succeed()) + }) + + itShouldHaveNetworkConnectivity( + func() *framework.PodInterface { return vms[0].vmPod }, + func() framework.Executor { return vms[0].ssh }) + }) +} + +type multiCNIVM struct { + vm *framework.VMInterface + vmPod *framework.PodInterface + ssh framework.Executor + ips [2]string +} + +func makeMultiCNIVM(name string, addCNIAnnotation bool) *multiCNIVM { + vm := controller.VM(name) + opts := VMOptions{} + if addCNIAnnotation { + opts.MultiCNI = "calico,flannel" + } + Expect(vm.Create(opts.applyDefaults(), time.Minute*5, nil)).To(Succeed()) + vmPod, err := vm.Pod() + Expect(err).NotTo(HaveOccurred()) + return &multiCNIVM{ + vm: vm, + vmPod: vmPod, + ssh: waitSSH(vm), + } +} + +func (mcv *multiCNIVM) ensureIPOnSecondEth() { + _, err := framework.RunSimple(mcv.ssh, ensureEth1UpCmd) + Expect(err).NotTo(HaveOccurred()) +} + +func (mcv *multiCNIVM) retrieveIPs() { + for i := 0; i < 2; i++ { + ip, err := framework.RunSimple(mcv.ssh, fmt.Sprintf(getLinkIpCmd, i)) + Expect(err).NotTo(HaveOccurred()) + mcv.ips[i] = ip + } +} + +func (mcv *multiCNIVM) teardown() { + if mcv.ssh != nil { + mcv.ssh.Close() + } + if mcv.vm != nil { + deleteVM(mcv.vm) + } +} + +func (mcv *multiCNIVM) ping(ip string) { + Expect(framework.RunSimple(mcv.ssh, fmt.Sprintf("ping -c1 %s", ip))). + To(MatchRegexp("1 .*transmitted, 1 .*received, 0% .*loss")) +} + +func (mcv *multiCNIVM) netcatListen(listenIp string) error { + _, err := framework.RunSimple(mcv.ssh, netcatListenCmd) + return err +} + +func (mcv *multiCNIVM) netcatConnect(targetIp string) { + Eventually(func() error { + _, err := framework.RunSimple(mcv.ssh, fmt.Sprintf(netcatSendCmd, targetIp)) + return err + }, 60).Should(Succeed()) +}