From 4e9685966084250ef92cf8223e77e475e1be1145 Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Thu, 11 Jul 2024 16:29:36 +0200 Subject: [PATCH] Add dualstack integration tests (#532) --- tests/integration/lxd-dualstack-profile.yaml | 7 ++ .../templates/bootstrap-dualstack.yaml | 16 +++ .../templates/nginx-dualstack.yaml | 2 +- tests/integration/tests/conftest.py | 4 +- tests/integration/tests/test_dualstack.py | 65 +++++++++++ tests/integration/tests/test_util/config.py | 14 +++ .../tests/test_util/harness/base.py | 4 +- .../tests/test_util/harness/juju.py | 5 +- .../tests/test_util/harness/local.py | 5 +- .../tests/test_util/harness/lxd.py | 103 ++++++++++++------ .../tests/test_util/harness/multipass.py | 7 +- 11 files changed, 192 insertions(+), 40 deletions(-) create mode 100644 tests/integration/lxd-dualstack-profile.yaml create mode 100644 tests/integration/templates/bootstrap-dualstack.yaml create mode 100644 tests/integration/tests/test_dualstack.py diff --git a/tests/integration/lxd-dualstack-profile.yaml b/tests/integration/lxd-dualstack-profile.yaml new file mode 100644 index 000000000..0d2df970c --- /dev/null +++ b/tests/integration/lxd-dualstack-profile.yaml @@ -0,0 +1,7 @@ +description: "LXD profile for Canonical Kubernetes with dualstack networking" +devices: + eth0: + name: eth0 + nictype: bridged + parent: LXD_DUALSTACK_NETWORK + type: nic diff --git a/tests/integration/templates/bootstrap-dualstack.yaml b/tests/integration/templates/bootstrap-dualstack.yaml new file mode 100644 index 000000000..548426403 --- /dev/null +++ b/tests/integration/templates/bootstrap-dualstack.yaml @@ -0,0 +1,16 @@ +cluster-config: + network: + enabled: true + dns: + enabled: true + cluster-domain: cluster.local + local-storage: + enabled: true + local-path: /storage/path + default: false + gateway: + enabled: true + metrics-server: + enabled: true +pod-cidr: 10.100.0.0/16,fd01::/108 +service-cidr: 10.200.0.0/16,fd98::/108 diff --git a/tests/integration/templates/nginx-dualstack.yaml b/tests/integration/templates/nginx-dualstack.yaml index 959fc2a11..fb15fa165 100644 --- a/tests/integration/templates/nginx-dualstack.yaml +++ b/tests/integration/templates/nginx-dualstack.yaml @@ -21,7 +21,7 @@ spec: apiVersion: v1 kind: Service metadata: - name: nginx6 + name: nginx-dualstack labels: run: nginxdualstack spec: diff --git a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py index 309cf982b..53a6055b4 100644 --- a/tests/integration/tests/conftest.py +++ b/tests/integration/tests/conftest.py @@ -50,8 +50,8 @@ def pytest_configure(config): config.addinivalue_line( "markers", "node_count: Mark a test to specify how many instance nodes need to be created\n" - "disable_k8s_bootstrapping: By default, the first k8s node is bootstrapped. This marker disables that." - "etcd_count: Mark a test to specify how many etcd instance nodes need to be created (None by default)", + "disable_k8s_bootstrapping: By default, the first k8s node is bootstrapped. This marker disables that.\n" + "etcd_count: Mark a test to specify how many etcd instance nodes need to be created (None by default)\n", ) diff --git a/tests/integration/tests/test_dualstack.py b/tests/integration/tests/test_dualstack.py new file mode 100644 index 000000000..cc6003cf2 --- /dev/null +++ b/tests/integration/tests/test_dualstack.py @@ -0,0 +1,65 @@ +# +# Copyright 2024 Canonical, Ltd. +# +import logging +from ipaddress import IPv4Address, IPv6Address, ip_address +from pathlib import Path + +import pytest +from test_util import config, harness, util + +LOG = logging.getLogger(__name__) + + +@pytest.mark.node_count(1) +def test_dualstack(h: harness.Harness, tmp_path: Path): + snap_path = (tmp_path / "k8s.snap").as_posix() + main = h.new_instance(dualstack=True) + util.setup_k8s_snap(main, snap_path) + + bootstrap_config = (config.MANIFESTS_DIR / "bootstrap-dualstack.yaml").read_text() + + main.exec( + ["k8s", "bootstrap", "--file", "-"], + input=str.encode(bootstrap_config), + ) + util.wait_until_k8s_ready(main, [main]) + + dualstack_config = (config.MANIFESTS_DIR / "nginx-dualstack.yaml").read_text() + + # Deploy nginx with dualstack service + main.exec( + ["k8s", "kubectl", "apply", "-f", "-"], input=str.encode(dualstack_config) + ) + addresses = ( + util.stubbornly(retries=5, delay_s=3) + .on(main) + .exec( + [ + "k8s", + "kubectl", + "get", + "svc", + "nginx-dualstack", + "-o", + "jsonpath='{.spec.clusterIPs[*]}'", + ], + text=True, + capture_output=True, + ) + .stdout + ) + + for ip in addresses.split(): + addr = ip_address(ip.strip("'")) + if isinstance(addr, IPv6Address): + address = f"http://[{str(addr)}]" + elif isinstance(addr, IPv4Address): + address = f"http://{str(addr)}" + else: + pytest.fail(f"Unknown IP address type: {addr}") + + # need to shell out otherwise this runs into permission errors + util.stubbornly(retries=3, delay_s=1).on(main).exec( + ["curl", address], shell=True + ) diff --git a/tests/integration/tests/test_util/config.py b/tests/integration/tests/test_util/config.py index 41d76b117..fc2de9401 100644 --- a/tests/integration/tests/test_util/config.py +++ b/tests/integration/tests/test_util/config.py @@ -37,6 +37,20 @@ or (DIR / ".." / ".." / "lxd-profile.yaml").read_text() ) +# LXD_DUALSTACK_NETWORK is the network to use for LXD containers with dualstack configured. +LXD_DUALSTACK_NETWORK = os.getenv("TEST_LXD_DUALSTACK_NETWORK") or "dualstack-br0" + +# LXD_DUALSTACK_PROFILE_NAME is the profile name to use for LXD containers with dualstack configured. +LXD_DUALSTACK_PROFILE_NAME = ( + os.getenv("TEST_LXD_DUALSTACK_PROFILE_NAME") or "k8s-integration-dualstack" +) + +# LXD_DUALSTACK_PROFILE is the profile to use for LXD containers with dualstack configured. +LXD_DUALSTACK_PROFILE = ( + os.getenv("TEST_LXD_DUALSTACK_PROFILE") + or (DIR / ".." / ".." / "lxd-dualstack-profile.yaml").read_text() +) + # LXD_IMAGE is the image to use for LXD containers. LXD_IMAGE = os.getenv("TEST_LXD_IMAGE") or "ubuntu:22.04" diff --git a/tests/integration/tests/test_util/harness/base.py b/tests/integration/tests/test_util/harness/base.py index 81a969a34..829d64511 100644 --- a/tests/integration/tests/test_util/harness/base.py +++ b/tests/integration/tests/test_util/harness/base.py @@ -42,10 +42,12 @@ class Harness: name: str - def new_instance(self) -> Instance: + def new_instance(self, dualstack: bool = False) -> Instance: """Creates a new instance on the infrastructure and returns an object which can be used to interact with it. + dualstack: If True, the instance will be created with dualstack support. + If the operation fails, a HarnessError is raised. """ raise NotImplementedError diff --git a/tests/integration/tests/test_util/harness/juju.py b/tests/integration/tests/test_util/harness/juju.py index 4d3a02bf6..d8e3a694c 100644 --- a/tests/integration/tests/test_util/harness/juju.py +++ b/tests/integration/tests/test_util/harness/juju.py @@ -53,7 +53,10 @@ def __init__(self): self.constraints, ) - def new_instance(self) -> Instance: + def new_instance(self, dualstack: bool = False) -> Instance: + if dualstack: + raise HarnessError("Dualstack is currently not supported by Juju harness") + for instance_id in self.existing_machines: if not self.existing_machines[instance_id]: LOG.debug("Reusing existing machine %s", instance_id) diff --git a/tests/integration/tests/test_util/harness/local.py b/tests/integration/tests/test_util/harness/local.py index 4a53730ce..2b790c6cf 100644 --- a/tests/integration/tests/test_util/harness/local.py +++ b/tests/integration/tests/test_util/harness/local.py @@ -27,10 +27,13 @@ def __init__(self): LOG.debug("Configured local substrate") - def new_instance(self) -> Instance: + def new_instance(self, dualstack: bool = False) -> Instance: if self.initialized: raise HarnessError("local substrate only supports up to one instance") + if dualstack: + raise HarnessError("Dualstack is currently not supported by Local harness") + self.initialized = True LOG.debug("Initializing instance") try: diff --git a/tests/integration/tests/test_util/harness/lxd.py b/tests/integration/tests/test_util/harness/lxd.py index a5aaebd8a..bc2c3909e 100644 --- a/tests/integration/tests/test_util/harness/lxd.py +++ b/tests/integration/tests/test_util/harness/lxd.py @@ -6,6 +6,7 @@ import shlex import subprocess from pathlib import Path +from typing import List from test_util import config from test_util.harness import Harness, HarnessError, Instance @@ -29,53 +30,52 @@ def __init__(self): self._next_id = 0 self.profile = config.LXD_PROFILE_NAME + self.dualstack_profile = None self.sideload_images_dir = config.LXD_SIDELOAD_IMAGES_DIR self.image = config.LXD_IMAGE self.instances = set() - LOG.debug("Checking for LXD profile %s", self.profile) - try: - run(["lxc", "profile", "show", self.profile]) - except subprocess.CalledProcessError: - try: - LOG.debug("Creating LXD profile %s", self.profile) - run(["lxc", "profile", "create", self.profile]) - - except subprocess.CalledProcessError as e: - raise HarnessError( - f"Failed to create LXD profile {self.profile}" - ) from e + self._configure_profile(self.profile, config.LXD_PROFILE) - try: - LOG.debug("Configuring LXD profile %s", self.profile) - run( - ["lxc", "profile", "edit", self.profile], - input=config.LXD_PROFILE.encode(), - ) - except subprocess.CalledProcessError as e: - raise HarnessError(f"Failed to configure LXD profile {self.profile}") from e + self._configure_network( + config.LXD_DUALSTACK_NETWORK, + "ipv4.address=auto", + "ipv6.address=auto", + "ipv4.nat=true", + "ipv6.nat=true", + ) + self.dualstack_profile = config.LXD_DUALSTACK_PROFILE_NAME + self._configure_profile( + self.dualstack_profile, + config.LXD_DUALSTACK_PROFILE.replace( + "LXD_DUALSTACK_NETWORK", config.LXD_DUALSTACK_NETWORK + ), + ) LOG.debug( "Configured LXD substrate (profile %s, image %s)", self.profile, self.image ) - def new_instance(self) -> Instance: + def new_instance(self, dualstack: bool = False) -> Instance: instance_id = f"k8s-integration-{os.urandom(3).hex()}-{self.next_id()}" LOG.debug("Creating instance %s with image %s", instance_id, self.image) + launch_lxd_command = [ + "lxc", + "launch", + self.image, + instance_id, + "-p", + "default", + "-p", + self.profile, + ] + + if dualstack: + launch_lxd_command.extend(["-p", self.dualstack_profile]) + try: - stubbornly(retries=3, delay_s=1).exec( - [ - "lxc", - "launch", - self.image, - instance_id, - "-p", - "default", - "-p", - self.profile, - ] - ) + stubbornly(retries=3, delay_s=1).exec(launch_lxd_command) self.instances.add(instance_id) if self.sideload_images_dir: @@ -108,6 +108,43 @@ def new_instance(self) -> Instance: self.exec(instance_id, ["snap", "wait", "system", "seed.loaded"]) return Instance(self, instance_id) + def _configure_profile(self, profile_name: str, profile_config: str): + LOG.debug("Checking for LXD profile %s", profile_name) + try: + run(["lxc", "profile", "show", profile_name]) + except subprocess.CalledProcessError: + try: + LOG.debug("Creating LXD profile %s", profile_name) + run(["lxc", "profile", "create", profile_name]) + + except subprocess.CalledProcessError as e: + raise HarnessError( + f"Failed to create LXD profile {profile_name}" + ) from e + + try: + LOG.debug("Configuring LXD profile %s", profile_name) + run( + ["lxc", "profile", "edit", profile_name], + input=profile_config.encode(), + ) + except subprocess.CalledProcessError as e: + raise HarnessError(f"Failed to configure LXD profile {profile_name}") from e + + def _configure_network(self, network_name: str, *network_args: List[str]): + LOG.debug("Checking for LXD network %s", network_name) + try: + run(["lxc", "network", "show", network_name]) + except subprocess.CalledProcessError: + try: + LOG.debug("Creating LXD network %s", network_name) + run(["lxc", "network", "create", network_name, *network_args]) + + except subprocess.CalledProcessError as e: + raise HarnessError( + f"Failed to create LXD network {network_name}" + ) from e + def send_file(self, instance_id: str, source: str, destination: str): if instance_id not in self.instances: raise HarnessError(f"unknown instance {instance_id}") diff --git a/tests/integration/tests/test_util/harness/multipass.py b/tests/integration/tests/test_util/harness/multipass.py index a98df7ea8..4cea3a194 100644 --- a/tests/integration/tests/test_util/harness/multipass.py +++ b/tests/integration/tests/test_util/harness/multipass.py @@ -36,7 +36,12 @@ def __init__(self): LOG.debug("Configured Multipass substrate (image %s)", self.image) - def new_instance(self) -> Instance: + def new_instance(self, dualstack: bool = False) -> Instance: + if dualstack: + raise HarnessError( + "Dualstack is currently not supported by Multipass harness" + ) + instance_id = f"k8s-integration-{os.urandom(3).hex()}-{self.next_id()}" LOG.debug("Creating instance %s with image %s", instance_id, self.image)