Add dualstack integration tests (#532)

tests/integration/lxd-dualstack-profile.yaml

@@ -0,0 +1,7 @@
+description: "LXD profile for Canonical Kubernetes with dualstack networking"
+devices:
+  eth0:
+    name: eth0
+    nictype: bridged
+    parent: LXD_DUALSTACK_NETWORK
+    type: nic

tests/integration/templates/bootstrap-dualstack.yaml

@@ -0,0 +1,16 @@
+cluster-config:
+  network:
+    enabled: true
+  dns:
+    enabled: true
+    cluster-domain: cluster.local
+  local-storage:
+    enabled: true
+    local-path: /storage/path
+    default: false
+  gateway:
+    enabled: true
+  metrics-server:
+    enabled: true
+pod-cidr: 10.100.0.0/16,fd01::/108
+service-cidr: 10.200.0.0/16,fd98::/108

tests/integration/templates/nginx-dualstack.yaml

@@ -21,7 +21,7 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: nginx6
+  name: nginx-dualstack
   labels:
     run: nginxdualstack
 spec:

tests/integration/tests/conftest.py

@@ -50,8 +50,8 @@ def pytest_configure(config):
     config.addinivalue_line(
         "markers",
         "node_count: Mark a test to specify how many instance nodes need to be created\n"
-        "disable_k8s_bootstrapping: By default, the first k8s node is bootstrapped. This marker disables that."
-        "etcd_count: Mark a test to specify how many etcd instance nodes need to be created (None by default)",
+        "disable_k8s_bootstrapping: By default, the first k8s node is bootstrapped. This marker disables that.\n"
+        "etcd_count: Mark a test to specify how many etcd instance nodes need to be created (None by default)\n",
     )
 
 

tests/integration/tests/test_dualstack.py

@@ -0,0 +1,65 @@
+#
+# Copyright 2024 Canonical, Ltd.
+#
+import logging
+from ipaddress import IPv4Address, IPv6Address, ip_address
+from pathlib import Path
+
+import pytest
+from test_util import config, harness, util
+
+LOG = logging.getLogger(__name__)
+
+
+@pytest.mark.node_count(1)
+def test_dualstack(h: harness.Harness, tmp_path: Path):
+    snap_path = (tmp_path / "k8s.snap").as_posix()
+    main = h.new_instance(dualstack=True)
+    util.setup_k8s_snap(main, snap_path)
+
+    bootstrap_config = (config.MANIFESTS_DIR / "bootstrap-dualstack.yaml").read_text()
+
+    main.exec(
+        ["k8s", "bootstrap", "--file", "-"],
+        input=str.encode(bootstrap_config),
+    )
+    util.wait_until_k8s_ready(main, [main])
+
+    dualstack_config = (config.MANIFESTS_DIR / "nginx-dualstack.yaml").read_text()
+
+    # Deploy nginx with dualstack service
+    main.exec(
+        ["k8s", "kubectl", "apply", "-f", "-"], input=str.encode(dualstack_config)
+    )
+    addresses = (
+        util.stubbornly(retries=5, delay_s=3)
+        .on(main)
+        .exec(
+            [
+                "k8s",
+                "kubectl",
+                "get",
+                "svc",
+                "nginx-dualstack",
+                "-o",
+                "jsonpath='{.spec.clusterIPs[*]}'",
+            ],
+            text=True,
+            capture_output=True,
+        )
+        .stdout
+    )
+
+    for ip in addresses.split():
+        addr = ip_address(ip.strip("'"))
+        if isinstance(addr, IPv6Address):
+            address = f"http://[{str(addr)}]"
+        elif isinstance(addr, IPv4Address):
+            address = f"http://{str(addr)}"
+        else:
+            pytest.fail(f"Unknown IP address type: {addr}")
+
+        # need to shell out otherwise this runs into permission errors
+        util.stubbornly(retries=3, delay_s=1).on(main).exec(
+            ["curl", address], shell=True
+        )

tests/integration/tests/test_util/config.py

@@ -37,6 +37,20 @@
     or (DIR / ".." / ".." / "lxd-profile.yaml").read_text()
 )
 
+# LXD_DUALSTACK_NETWORK is the network to use for LXD containers with dualstack configured.
+LXD_DUALSTACK_NETWORK = os.getenv("TEST_LXD_DUALSTACK_NETWORK") or "dualstack-br0"
+
+# LXD_DUALSTACK_PROFILE_NAME is the profile name to use for LXD containers with dualstack configured.
+LXD_DUALSTACK_PROFILE_NAME = (
+    os.getenv("TEST_LXD_DUALSTACK_PROFILE_NAME") or "k8s-integration-dualstack"
+)
+
+# LXD_DUALSTACK_PROFILE is the profile to use for LXD containers with dualstack configured.
+LXD_DUALSTACK_PROFILE = (
+    os.getenv("TEST_LXD_DUALSTACK_PROFILE")
+    or (DIR / ".." / ".." / "lxd-dualstack-profile.yaml").read_text()
+)
+
 # LXD_IMAGE is the image to use for LXD containers.
 LXD_IMAGE = os.getenv("TEST_LXD_IMAGE") or "ubuntu:22.04"
 

tests/integration/tests/test_util/harness/base.py

@@ -42,10 +42,12 @@ class Harness:
 
     name: str
 
-    def new_instance(self) -> Instance:
+    def new_instance(self, dualstack: bool = False) -> Instance:
         """Creates a new instance on the infrastructure and returns an object
         which can be used to interact with it.
 
+        dualstack: If True, the instance will be created with dualstack support.
+
         If the operation fails, a HarnessError is raised.
         """
         raise NotImplementedError

tests/integration/tests/test_util/harness/juju.py

@@ -53,7 +53,10 @@ def __init__(self):
                 self.constraints,
             )
 
-    def new_instance(self) -> Instance:
+    def new_instance(self, dualstack: bool = False) -> Instance:
+        if dualstack:
+            raise HarnessError("Dualstack is currently not supported by Juju harness")
+
         for instance_id in self.existing_machines:
             if not self.existing_machines[instance_id]:
                 LOG.debug("Reusing existing machine %s", instance_id)

tests/integration/tests/test_util/harness/local.py

@@ -27,10 +27,13 @@ def __init__(self):
 
         LOG.debug("Configured local substrate")
 
-    def new_instance(self) -> Instance:
+    def new_instance(self, dualstack: bool = False) -> Instance:
         if self.initialized:
             raise HarnessError("local substrate only supports up to one instance")
 
+        if dualstack:
+            raise HarnessError("Dualstack is currently not supported by Local harness")
+
         self.initialized = True
         LOG.debug("Initializing instance")
         try:

tests/integration/tests/test_util/harness/lxd.py

@@ -6,6 +6,7 @@
 import shlex
 import subprocess
 from pathlib import Path
+from typing import List
 
 from test_util import config
 from test_util.harness import Harness, HarnessError, Instance
@@ -29,53 +30,52 @@ def __init__(self):
         self._next_id = 0
 
         self.profile = config.LXD_PROFILE_NAME
+        self.dualstack_profile = None
         self.sideload_images_dir = config.LXD_SIDELOAD_IMAGES_DIR
         self.image = config.LXD_IMAGE
         self.instances = set()
 
-        LOG.debug("Checking for LXD profile %s", self.profile)
-        try:
-            run(["lxc", "profile", "show", self.profile])
-        except subprocess.CalledProcessError:
-            try:
-                LOG.debug("Creating LXD profile %s", self.profile)
-                run(["lxc", "profile", "create", self.profile])
-
-            except subprocess.CalledProcessError as e:
-                raise HarnessError(
-                    f"Failed to create LXD profile {self.profile}"
-                ) from e
+        self._configure_profile(self.profile, config.LXD_PROFILE)
 
-        try:
-            LOG.debug("Configuring LXD profile %s", self.profile)
-            run(
-                ["lxc", "profile", "edit", self.profile],
-                input=config.LXD_PROFILE.encode(),
-            )
-        except subprocess.CalledProcessError as e:
-            raise HarnessError(f"Failed to configure LXD profile {self.profile}") from e
+        self._configure_network(
+            config.LXD_DUALSTACK_NETWORK,
+            "ipv4.address=auto",
+            "ipv6.address=auto",
+            "ipv4.nat=true",
+            "ipv6.nat=true",
+        )
+        self.dualstack_profile = config.LXD_DUALSTACK_PROFILE_NAME
+        self._configure_profile(
+            self.dualstack_profile,
+            config.LXD_DUALSTACK_PROFILE.replace(
+                "LXD_DUALSTACK_NETWORK", config.LXD_DUALSTACK_NETWORK
+            ),
+        )
 
         LOG.debug(
             "Configured LXD substrate (profile %s, image %s)", self.profile, self.image
         )
 
-    def new_instance(self) -> Instance:
+    def new_instance(self, dualstack: bool = False) -> Instance:
         instance_id = f"k8s-integration-{os.urandom(3).hex()}-{self.next_id()}"
 
         LOG.debug("Creating instance %s with image %s", instance_id, self.image)
+        launch_lxd_command = [
+            "lxc",
+            "launch",
+            self.image,
+            instance_id,
+            "-p",
+            "default",
+            "-p",
+            self.profile,
+        ]
+
+        if dualstack:
+            launch_lxd_command.extend(["-p", self.dualstack_profile])
+
         try:
-            stubbornly(retries=3, delay_s=1).exec(
-                [
-                    "lxc",
-                    "launch",
-                    self.image,
-                    instance_id,
-                    "-p",
-                    "default",
-                    "-p",
-                    self.profile,
-                ]
-            )
+            stubbornly(retries=3, delay_s=1).exec(launch_lxd_command)
             self.instances.add(instance_id)
 
             if self.sideload_images_dir:
@@ -108,6 +108,43 @@ def new_instance(self) -> Instance:
         self.exec(instance_id, ["snap", "wait", "system", "seed.loaded"])
         return Instance(self, instance_id)
 
+    def _configure_profile(self, profile_name: str, profile_config: str):
+        LOG.debug("Checking for LXD profile %s", profile_name)
+        try:
+            run(["lxc", "profile", "show", profile_name])
+        except subprocess.CalledProcessError:
+            try:
+                LOG.debug("Creating LXD profile %s", profile_name)
+                run(["lxc", "profile", "create", profile_name])
+
+            except subprocess.CalledProcessError as e:
+                raise HarnessError(
+                    f"Failed to create LXD profile {profile_name}"
+                ) from e
+
+        try:
+            LOG.debug("Configuring LXD profile %s", profile_name)
+            run(
+                ["lxc", "profile", "edit", profile_name],
+                input=profile_config.encode(),
+            )
+        except subprocess.CalledProcessError as e:
+            raise HarnessError(f"Failed to configure LXD profile {profile_name}") from e
+
+    def _configure_network(self, network_name: str, *network_args: List[str]):
+        LOG.debug("Checking for LXD network %s", network_name)
+        try:
+            run(["lxc", "network", "show", network_name])
+        except subprocess.CalledProcessError:
+            try:
+                LOG.debug("Creating LXD network %s", network_name)
+                run(["lxc", "network", "create", network_name, *network_args])
+
+            except subprocess.CalledProcessError as e:
+                raise HarnessError(
+                    f"Failed to create LXD network {network_name}"
+                ) from e
+
     def send_file(self, instance_id: str, source: str, destination: str):
         if instance_id not in self.instances:
             raise HarnessError(f"unknown instance {instance_id}")

tests/integration/tests/test_util/harness/multipass.py

@@ -36,7 +36,12 @@ def __init__(self):
 
         LOG.debug("Configured Multipass substrate (image %s)", self.image)
 
-    def new_instance(self) -> Instance:
+    def new_instance(self, dualstack: bool = False) -> Instance:
+        if dualstack:
+            raise HarnessError(
+                "Dualstack is currently not supported by Multipass harness"
+            )
+
         instance_id = f"k8s-integration-{os.urandom(3).hex()}-{self.next_id()}"
 
         LOG.debug("Creating instance %s with image %s", instance_id, self.image)