Performance Test workload (2) (#199)

test/performance/Readme.md

@@ -27,17 +27,6 @@ In general, all end to end tests will require specifying the local path to the s
 
 End to end tests are typically run with: `cd test/performance && tox -e performance`
 
-### Running end to end tests on the local machine
-
-```bash
-export TEST_SNAP=$PWD/k8s.snap
-export TEST_SUBSTRATE=local
-
-cd test/performance && tox -e performance
-```
-
-> *NOTE*: When running locally, end to end tests that create more than one instance will fail.
-
 ### Running end to end tests on LXD containers
 
 First, make sure that you have initialized LXD:

test/performance/templates/api-intensive.yaml

@@ -0,0 +1,29 @@
+global:
+  writeToFile: true
+  metricsDirectory: collected-metrics
+
+jobs:
+  - name: api-intensive
+    jobIterations: 100
+    qps: 500
+    burst: 500
+    namespacedIterations: true
+    namespace: api-intensive
+    podWait: false
+    cleanup: true
+    waitFor: []
+    waitWhenFinished: true
+    objects:
+      - objectTemplate: configmap.yaml
+        replicas: 10
+      - objectTemplate: secret.yaml
+        replicas: 10
+  - name: remove-configmaps-secrets
+    qps: 100
+    burst: 100
+    jobType: delete
+    objects:
+      - kind: ConfigMap
+        labelSelector: { kube-burner-job: api-intensive }
+      - kind: Secret
+        labelSelector: { kube-burner-job: api-intensive }

test/performance/templates/bootstrap-session.yaml

@@ -0,0 +1,17 @@
+# Contains the bootstrap configuration for the session instance of the integration tests.
+# The session instance persists over test runs and is used to speed-up the integration tests.
+cluster-config:
+  network:
+    enabled: true
+  dns:
+    enabled: true
+  ingress:
+    enabled: true
+  load-balancer:
+    enabled: true
+  local-storage:
+    enabled: true
+  gateway:
+    enabled: true
+  metrics-server:
+    enabled: true

test/performance/templates/configmap.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: configmap-{{.Replica}}
+data:
+  data.yaml: |-
+    a: 1
+    b: 2
+    c: 3
+    d: "ZviE5QVws2zuKJZMCSkRp0hqHGeP6GgqPRZ8B7l48EwGMhnRAjuNiDhjlpUlFM7comB1ipEEQNf69FeMFXEkSI9ukRVP559Ug497E7u8miZqBzuuo5oXw8zKH2WHz2OtP5uQDdm9x6G4Ki7Lwo84p0Yc82wEJPbJpM2SvO5ze0WOi3jHWqDbmkORhEF2RWBqfe3uzsABU7gi86n4VaFCuD2sFnwVsSjJWDm7psVoMpJWQbBRpzjKhnDHbYXjkobk9gTH7mFxImxGfoQsW716auNO0ATJ6C4SoTdZ1CUngjLD1bkBy35AbfzjIjQGxjrE2gJ0qeBKZo0uwYnAFraCYMZ6uhNQ6B1dO9bELze3TQiQv8CrC4bplhCth62E2lAz6Q095HFbOYKyUeFp0AArEN0LlIxW6JqwV4ssvA1LSt1HnnsYVc0yZO6kFuR5sNRuPxqc01AArNQ4BP9UudQdVagJ15MIcrGhBqpVFoSIdyK2w4eZQwLpZxpXdJCfdSdkJwWND42O07oVkbNt1kpKFJ8M3clYjpZlo5zkRjM90acXiVU7FNn5D6yQkureBO4y2AkYYVkj35VvijvY8VBkEveMiGU4ztsaOR7Px5UsvgPdgvZG2IOUYezfzcF5nyRVwYTQWFuXae3rGHOXi6j0KmAQix2xnezIMPGJSDQPROzbiPYhvtXo2TdAmhJd1DgEyi9lbLrwJ3rwCkZbKl9RsfxoihHnpWWH24wCKizJfSfcXmNR6Bvaj9G2Ldk7ppLBFpwMyxK8w4u3LrduvvRDaKcuQKwopnpRKG4njJMITRrynlprWBWyWaiclnNBxqtYCWYwbyqFwAjg8snkwqFmEhkYlp6kuzEEF8Naz0KVVMRTyigUQYVKUh10Ps8q98YrXpmOqnY2lVXE6n54lraVHIwFURstm6ROCSL7bThyKxXO0SGE4ioGOUER3aSNGlyiFlFaEHFdZ69rjHwlY5lgjMUysifFDKz9oLNssPZwFD6wxiFmtR3y5dtzJPtajS4bf6wnz5Jn6kzvPtcNyAW9fRdUmDvhEDCWBtDT0vP59BQ8zlwKvnHMiJwrGL0xG14v6CFQJ4Vv4K7WFFySkRmQjP6XiVYieiFDUC9G0RQ0XD3RB1XmWxNXl1Q9hDq6XRyBqRBDhb0rlQK9QUNnAm772lES2kKhaXHXTigKU2YOQsUydTXBLiJOsRYdfvLvEUIXp60xlSYkKah3SGfBMYYWadJv7l2pmHAHRNWhiUXsffnr6llYmRbJMXuTZ5rjq1M5WVXGb5AdlkLMOh7l931Tkgb5GaMskLFapd9DkhsimadvLWJTRFSPLoyLxybJqlnZKiRNY9sQ9TF9OnNMEs7pfZPaRx5RyACVykzyRs9AWMlgKP6X8KExXUrWb7f3E3GNXqWaoO23KL7oxY0BeWZTBKfT0HofhXnBhyv6HZ8TUPQRJifTlZ2Q6v53EPvVey6u9oMcEaxqTprcf1FfGa0TgagtewO9aJ5UOH9yPPYerk7GUBI5s5nlv3uSlJJAZXBuZmSWz9zQ6anmRkykTYpDKGKE6R22P0FL1b5bsuy4p5z30aBD58VQ1nr5OhMnM7pqpKORoLRYCCKAl7SH7lxGvrdlb3VRT8CkBWfPgYmHPPgymiFOVeZUIZdpJpj4Ti7vRLzhtjxCxnqXm7famDTA9zLBWtbmKfWncsWIW672EIE37NAcydhwmWXKFiMutmnlhzgizpLJHPNcKgCpBpLYykz1T4ISEyKy499RY5rU1I2pRSZDO6ciUyVybURadqILARY44m61kX5CU1q4ovjrtawXl3pVq78azyHrbEDFNGrkBHPbXdr0KdcrbBXLkm09Rs5IpADjOTXQqjkujScwiIPhutkDdey1lTaUlVAtKMzBBXcjxm16TN1BaaZjuQxUGNDxVYMJoWIjFMv7nD24ZTA9nPjli2lQk2ZIueXY2WnVMNL3xxCzG4eX86Gb2iqFD631vLr0XfhsUBXrJBY1F1pX4M3qSWK93XAQt5DEPhvmdFU8PQgTu5Pj9AISDYAv"

test/performance/templates/secret.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: secret-{{.Replica}}
+type: Opaque
+data:
+  password: Zm9vb29vb29vb29vb29vbwo=

test/performance/tests/conftest.py

@@ -1,5 +1,6 @@
 #
-# Copyright 2024 Canonical, Ltd.#
+# Copyright 2024 Canonical, Ltd.
+#
 import itertools
 import logging
 from pathlib import Path
@@ -187,3 +188,44 @@ def instances(
             _generate_inspection_report(h, instance.id)
 
         h.delete_instance(instance.id)
+
+
+@pytest.fixture(scope="session")
+def session_instance(
+    h: harness.Harness, tmp_path_factory: pytest.TempPathFactory, request
+) -> Generator[harness.Instance, None, None]:
+    """Constructs and bootstraps an instance that persists over a test session.
+
+    Bootstraps the instance with all k8sd features enabled to reduce testing time.
+    """
+    LOG.info("Setup node and enable all features")
+
+    tmp_path = tmp_path_factory.mktemp("data")
+    instance = h.new_instance()
+    snap = next(snap_versions(request))
+    util.setup_k8s_snap(instance, tmp_path, snap)
+
+    bootstrap_config_path = "/root/bootstrap-session.yaml"
+    instance.send_file(
+        (config.MANIFESTS_DIR / "bootstrap-session.yaml").as_posix(),
+        bootstrap_config_path,
+    )
+
+    instance_default_ip = util.get_default_ip(instance)
+
+    instance.exec(["k8s", "bootstrap", "--file", bootstrap_config_path])
+    instance_default_cidr = util.get_default_cidr(instance, instance_default_ip)
+
+    lb_cidr = util.find_suitable_cidr(
+        parent_cidr=instance_default_cidr,
+        excluded_ips=[instance_default_ip],
+    )
+
+    instance.exec(
+        ["k8s", "set", f"load-balancer.cidrs={lb_cidr}", "load-balancer.l2-mode=true"]
+    )
+    util.wait_until_k8s_ready(instance, [instance])
+    util.wait_for_network(instance)
+    util.wait_for_dns(instance)
+
+    yield instance

test/performance/tests/test_performance.py → test/performance/tests/test_multi_node.py

@@ -1,16 +1,17 @@
 #
-# Copyright 2024 Canonical, Ltd.#
+# Copyright 2024 Canonical, Ltd.
+#
 import logging
 from typing import List
 
 import pytest
-from test_util import harness, util
+from test_util import harness, metrics, util
 
 LOG = logging.getLogger(__name__)
 
 
 @pytest.mark.node_count(3)
-def test_load_test(instances: List[harness.Instance]):
+def test_three_node_load(instances: List[harness.Instance]):
     cluster_node = instances[0]
     joining_node = instances[1]
     joining_node_2 = instances[2]
@@ -30,3 +31,9 @@ def test_load_test(instances: List[harness.Instance]):
     assert "control-plane" in util.get_local_node_status(cluster_node)
     assert "control-plane" in util.get_local_node_status(joining_node)
     assert "control-plane" in util.get_local_node_status(joining_node_2)
+
+    metrics.configure_kube_burner(cluster_node)
+    process_dict = metrics.collect_metrics(instances)
+    metrics.run_kube_burner(cluster_node)
+    metrics.stop_metrics(instances, process_dict)
+    metrics.pull_metrics(instances)

test/performance/tests/test_single_node.py

@@ -0,0 +1,17 @@
+#
+# Copyright 2024 Canonical, Ltd.
+#
+import logging
+
+from test_util import harness, metrics
+
+LOG = logging.getLogger(__name__)
+
+
+def test_single_node_load(session_instance: harness.Instance):
+    """Test the performance of a single node cluster with all features enabled."""
+    metrics.configure_kube_burner(session_instance)
+    process_dict = metrics.collect_metrics([session_instance])
+    metrics.run_kube_burner(session_instance)
+    metrics.stop_metrics([session_instance], process_dict)
+    metrics.pull_metrics([session_instance])

test/performance/tests/test_util/config.py

@@ -1,5 +1,6 @@
 #
-# Copyright 2024 Canonical, Ltd.#
+# Copyright 2024 Canonical, Ltd.
+#
 import os
 from pathlib import Path
 
@@ -25,6 +26,12 @@
 # SNAP_NAME is the name of the snap under test.
 SNAP_NAME = os.getenv("TEST_SNAP_NAME") or "k8s"
 
+# KUBE_BURNER_URL is the version of kube-burner to use.
+KUBE_BURNER_URL = (
+    os.getenv("TEST_KUBE_BURNER_URL")
+    or "https://github.com/kube-burner/kube-burner/releases/download/v1.2/kube-burner-1.2-Linux-x86_64.tar.gz"
+)
+
 # FLAVOR is the flavour to use for running the performance tests.
 FLAVOR = os.getenv("TEST_FLAVOR") or ""
 

test/performance/tests/test_util/harness/__init__.py

@@ -1,5 +1,6 @@
 #
-# Copyright 2024 Canonical, Ltd.#
+# Copyright 2024 Canonical, Ltd.
+#
 from test_util.harness.base import Harness, HarnessError, Instance
 from test_util.harness.lxd import LXDHarness
 

test/performance/tests/test_util/harness/base.py

@@ -1,5 +1,6 @@
 #
-# Copyright 2024 Canonical, Ltd.#
+# Copyright 2024 Canonical, Ltd.
+#
 import subprocess
 from functools import cached_property, partial
 
@@ -84,7 +85,7 @@ def pull_file(self, instance_id: str, source: str, destination: str):
         raise NotImplementedError
 
     def exec(
-        self, instance_id: str, command: list, **kwargs
+        self, instance_id: str, command: list, background: bool = False, **kwargs
     ) -> subprocess.CompletedProcess:
         """Run a command as root on the instance.
 

test/performance/tests/test_util/harness/lxd.py

@@ -1,5 +1,6 @@
 #
-# Copyright 2024 Canonical, Ltd.#
+# Copyright 2024 Canonical, Ltd.
+#
 import logging
 import os
 import shlex
@@ -9,7 +10,7 @@
 
 from test_util import config
 from test_util.harness import Harness, HarnessError, Instance
-from test_util.util import run, stubbornly
+from test_util.util import run, run_popen, stubbornly
 
 LOG = logging.getLogger(__name__)
 
@@ -155,13 +156,23 @@ def pull_file(self, instance_id: str, source: str, destination: str):
         except subprocess.CalledProcessError as e:
             raise HarnessError("lxc file push command failed") from e
 
-    def exec(self, instance_id: str, command: list, **kwargs):
+    def exec(self, instance_id: str, command: list, background: bool = False, **kwargs):
         if instance_id not in self.instances:
             raise HarnessError(f"unknown instance {instance_id}")
 
         LOG.debug("Execute command %s in instance %s", command, instance_id)
+
+        if ">" in " ".join(command):
+            command_str = " ".join(command)
+        else:
+            command_str = shlex.join(command)
+        if background:
+            return run_popen(
+                ["lxc", "shell", instance_id, "--", "bash", "-c", command_str],
+                **kwargs,
+            )
         return run(
-            ["lxc", "shell", instance_id, "--", "bash", "-c", shlex.join(command)],
+            ["lxc", "shell", instance_id, "--", "bash", "-c", command_str],
             **kwargs,
         )
 

test/performance/tests/test_util/metrics.py

@@ -0,0 +1,83 @@
+#
+# Copyright 2024 Canonical, Ltd.
+#
+from typing import List
+
+from test_util import config, harness, util
+
+
+def stop_metrics(instances: List[harness.Instance], process_dict: dict):
+    """Stops collecting metrics in the background from each instance."""
+    for instance in instances:
+        process_dict[instance.id].kill()
+
+
+def collect_metrics(instances: List[harness.Instance]):
+    """
+    Starts collecting metrics in the background from each instance. Returns a dictionary
+    with the process object for each instance.
+    """
+    process_dict = {}
+    for instance in instances:
+        pid = instance.exec(
+            ["pgrep", "k8s-dqlite"], text=True, capture_output=True
+        ).stdout.strip()
+        util.stubbornly(retries=5, delay_s=3).on(instance).exec(
+            ["apt-get", "install", "-y", "sysstat"]
+        )
+        subprocess = instance.exec(
+            [
+                "pidstat",
+                "-druh",
+                "-p",
+                pid,
+                "1",
+                ">",
+                f"/root/{instance.id}_metrics.log",
+            ],
+            background=True,
+        )
+        process_dict[instance.id] = subprocess
+    return process_dict
+
+
+def pull_metrics(instances: List[harness.Instance]):
+    """Pulls metrics file from each instance to the local machine."""
+    for instance in instances:
+        instance.pull_file(
+            f"/root/{instance.id}_metrics.log", f"./{instance.id}_metrics.log"
+        )
+
+
+def configure_kube_burner(instance: harness.Instance):
+    """Downloads and sets up `kube-burner` on each instance if it's not already present."""
+    if (
+        not instance.exec(["test", "-f", "/root/kube-burner"], check=False).returncode
+        == 0
+    ):
+        url = config.KUBE_BURNER_URL
+        instance.exec(["wget", url])
+        instance.exec(
+            ["tar", "-zxvf", "kube-burner-1.2-Linux-x86_64.tar.gz", "kube-burner"]
+        )
+        instance.exec(["rm", "kube-burner-1.2-Linux-x86_64.tar.gz"])
+        instance.exec(["chmod", "+x", "/root/kube-burner"])
+
+    instance.exec(["mkdir", "-p", "/root/templates"])
+    instance.send_file(
+        (config.MANIFESTS_DIR / "api-intensive.yaml").as_posix(),
+        "/root/api-intensive.yaml",
+    )
+    instance.send_file(
+        (config.MANIFESTS_DIR / "secret.yaml").as_posix(), "/root/secret.yaml"
+    )
+    instance.send_file(
+        (config.MANIFESTS_DIR / "configmap.yaml").as_posix(), "/root/configmap.yaml"
+    )
+
+
+def run_kube_burner(instance: harness.Instance):
+    """Copies kubeconfig and runs kube-burner on the instance."""
+    instance.exec(["mkdir", "-p", "/root/.kube"])
+    instance.exec(["k8s", "config", ">", "/root/.kube/config"])
+    instance.exec(["/root/kube-burner", "init", "-c", "/root/api-intensive.yaml"])