From 6ce90fa76c02c1fa8adbcc4b011ab63aa6bbe461 Mon Sep 17 00:00:00 2001 From: Benjamin Schimke Date: Tue, 24 Sep 2024 08:55:50 +0200 Subject: [PATCH] Add version upgrade tests (#678) --------- Co-authored-by: Adam Dyess --- .github/workflows/integration-informing.yaml | 15 +-- .github/workflows/integration.yaml | 15 +-- .github/workflows/nightly-test.yaml | 11 ++- tests/integration/tests/conftest.py | 12 ++- tests/integration/tests/test_util/config.py | 7 ++ tests/integration/tests/test_util/snap.py | 93 +++++++++++++++++++ .../tests/test_version_upgrades.py | 54 +++++++++++ 7 files changed, 188 insertions(+), 19 deletions(-) create mode 100644 tests/integration/tests/test_util/snap.py create mode 100644 tests/integration/tests/test_version_upgrades.py diff --git a/.github/workflows/integration-informing.yaml b/.github/workflows/integration-informing.yaml index 0046b959b..344ffb6e4 100644 --- a/.github/workflows/integration-informing.yaml +++ b/.github/workflows/integration-informing.yaml @@ -80,16 +80,17 @@ jobs: uses: actions/download-artifact@v4 with: name: k8s-${{ matrix.patch }}.snap - path: build + path: ${{ github.workspace }}/build - name: Apply ${{ matrix.patch }} patch run: | ./build-scripts/patches/${{ matrix.patch }}/apply - name: Run end to end tests + env: + TEST_SNAP: ${{ github.workspace }}/build/k8s-${{ matrix.patch }}.snap + TEST_SUBSTRATE: lxd + TEST_LXD_IMAGE: ${{ matrix.os }} + TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports run: | - export TEST_SNAP="$PWD/build/k8s-${{ matrix.patch }}.snap" - export TEST_SUBSTRATE=lxd - export TEST_LXD_IMAGE=${{ matrix.os }} - export TEST_INSPECTION_REPORTS_DIR="$HOME/inspection-reports" # IPv6-only is only supported on moonray if [[ "${{ matrix.patch }}" == "moonray" ]]; then export TEST_IPV6_ONLY="true" @@ -98,11 +99,11 @@ jobs: - name: Prepare inspection reports if: failure() run: | - tar -czvf inspection-reports.tar.gz -C $HOME inspection-reports + tar -czvf inspection-reports.tar.gz -C ${{ github.workspace }} inspection-reports echo "artifact_name=inspection-reports-${{ matrix.os }}-${{ matrix.patch }}" | sed 's/:/-/g' >> $GITHUB_ENV - name: Upload inspection report artifact if: failure() uses: actions/upload-artifact@v4 with: name: ${{ env.artifact_name }} - path: inspection-reports.tar.gz + path: ${{ github.workspace }}/inspection-reports.tar.gz diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index e1f4543e9..d2589b8ca 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -95,25 +95,26 @@ jobs: uses: actions/download-artifact@v4 with: name: k8s.snap - path: build + path: ${{ github.workspace }}/build - name: Run end to end tests + env: + TEST_SNAP: ${{ github.workspace }}/build/k8s.snap + TEST_SUBSTRATE: lxd + TEST_LXD_IMAGE: ${{ matrix.os }} + TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports run: | - export TEST_SNAP="$PWD/build/k8s.snap" - export TEST_SUBSTRATE=lxd - export TEST_LXD_IMAGE=${{ matrix.os }} - export TEST_INSPECTION_REPORTS_DIR="$HOME/inspection-reports" cd tests/integration && sg lxd -c 'tox -e integration' - name: Prepare inspection reports if: failure() run: | - tar -czvf inspection-reports.tar.gz -C $HOME inspection-reports + tar -czvf inspection-reports.tar.gz -C ${{ github.workspace }} inspection-reports echo "artifact_name=inspection-reports-${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV - name: Upload inspection report artifact if: failure() uses: actions/upload-artifact@v4 with: name: ${{ env.artifact_name }} - path: inspection-reports.tar.gz + path: ${{ github.workspace }}/inspection-reports.tar.gz security-scan: permissions: diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml index cd661474f..ea00bcd4f 100644 --- a/.github/workflows/nightly-test.yaml +++ b/.github/workflows/nightly-test.yaml @@ -42,9 +42,14 @@ jobs: cd build snap download k8s --channel=${{ matrix.releases }} --basename k8s - name: Run end to end tests # tox path needs to be specified for arm64 + env: + TEST_SNAP: ${{ github.workspace }}/build/k8s-${{ matrix.patch }}.snap + TEST_SUBSTRATE: lxd + TEST_LXD_IMAGE: ${{ matrix.os }} + TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports + # Test the latest (up to) 6 releases for the flavour + # TODO(ben): upgrade nightly to run all flavours + TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic" run: | - export TEST_SNAP="$PWD/build/k8s.snap" - export TEST_SUBSTRATE=lxd - export TEST_LXD_IMAGE="${{ matrix.os }}" export PATH="/home/runner/.local/bin:$PATH" cd tests/integration && sg lxd -c 'tox -e integration' diff --git a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py index eb6fbb07f..98e170fae 100644 --- a/tests/integration/tests/conftest.py +++ b/tests/integration/tests/conftest.py @@ -83,6 +83,7 @@ def pytest_configure(config): "markers", "bootstrap_config: Provide a custom bootstrap config to the bootstrapping node.\n" "disable_k8s_bootstrapping: By default, the first k8s node is bootstrapped. This marker disables that.\n" + "no_setup: No setup steps (pushing snap, bootstrapping etc.) are performed on any node for this test.\n" "dualstack: Support dualstack on the instances.\n" "etcd_count: Mark a test to specify how many etcd instance nodes need to be created (None by default)\n" "node_count: Mark a test to specify how many instance nodes need to be created\n", @@ -103,6 +104,11 @@ def disable_k8s_bootstrapping(request) -> bool: return bool(request.node.get_closest_marker("disable_k8s_bootstrapping")) +@pytest.fixture(scope="function") +def no_setup(request) -> bool: + return bool(request.node.get_closest_marker("no_setup")) + + @pytest.fixture(scope="function") def bootstrap_config(request) -> Union[str, None]: bootstrap_config_marker = request.node.get_closest_marker("bootstrap_config") @@ -123,6 +129,7 @@ def instances( node_count: int, tmp_path: Path, disable_k8s_bootstrapping: bool, + no_setup: bool, bootstrap_config: Union[str, None], dualstack: bool, ) -> Generator[List[harness.Instance], None, None]: @@ -145,9 +152,10 @@ def instances( # Create instances and setup the k8s snap in each. instance = h.new_instance(dualstack=dualstack) instances.append(instance) - util.setup_k8s_snap(instance, snap_path) + if not no_setup: + util.setup_k8s_snap(instance, snap_path) - if not disable_k8s_bootstrapping: + if not disable_k8s_bootstrapping and not no_setup: first_node, *_ = instances if bootstrap_config is not None: diff --git a/tests/integration/tests/test_util/config.py b/tests/integration/tests/test_util/config.py index 2778d893f..b9e535683 100644 --- a/tests/integration/tests/test_util/config.py +++ b/tests/integration/tests/test_util/config.py @@ -88,3 +88,10 @@ # JUJU_MACHINES is a list of existing Juju machines to use. JUJU_MACHINES = os.getenv("TEST_JUJU_MACHINES") or "" + +# A list of space-separated channels for which the upgrade tests should be run in sequential order. +# First entry is the bootstrap channel. Afterwards, upgrades are done in order. +# Alternatively, use 'recent ' to get the latest channels for . +VERSION_UPGRADE_CHANNELS = ( + os.environ.get("TEST_VERSION_UPGRADE_CHANNELS", "").strip().split() +) diff --git a/tests/integration/tests/test_util/snap.py b/tests/integration/tests/test_util/snap.py new file mode 100644 index 000000000..9ad77ab49 --- /dev/null +++ b/tests/integration/tests/test_util/snap.py @@ -0,0 +1,93 @@ +# +# Copyright 2024 Canonical, Ltd. +# +import json +import logging +import re +import urllib.error +import urllib.request +from typing import List + +LOG = logging.getLogger(__name__) + +SNAP_NAME = "k8s" + +# For Snap Store API request +SNAPSTORE_INFO_API = "https://api.snapcraft.io/v2/snaps/info/" +SNAPSTORE_HEADERS = { + "Snap-Device-Series": "16", + "User-Agent": "Mozilla/5.0", +} +RISK_LEVELS = ["stable", "candidate", "beta", "edge"] + + +def get_snap_info(snap_name=SNAP_NAME): + """Get the snap info from the Snap Store API.""" + req = urllib.request.Request( + SNAPSTORE_INFO_API + snap_name, headers=SNAPSTORE_HEADERS + ) + try: + with urllib.request.urlopen(req) as response: # nosec + return json.loads(response.read().decode()) + except urllib.error.HTTPError as e: + LOG.exception("HTTPError ({%s}): {%s} {%s}", req.full_url, e.code, e.reason) + raise + except urllib.error.URLError as e: + LOG.exception("URLError ({%s}): {%s}", req.full_url, e.reason) + raise + + +def get_latest_channels( + num_of_channels: int, flavor: str, arch: str, include_latest=True +) -> List[str]: + """Get an ascending list of latest channels based on the number of channels and flavour. + + e.g. get_latest_release_channels(3, "classic") -> ['1.31-classic/candidate', '1.30-classic/candidate'] + if there are less than num_of_channels available, return all available channels. + Only the most stable risk level is returned for each major.minor version. + By default, the `latest/edge/` channel is included in the list. + """ + snap_info = get_snap_info() + + # Extract channel information + channels = snap_info.get("channel-map", []) + available_channels = [ + ch["channel"]["name"] + for ch in channels + if ch["channel"]["architecture"] == arch + ] + + # Define regex pattern to match channels in the format 'major.minor-flavour' + if flavor == "strict": + pattern = re.compile(r"(\d+)\.(\d+)\/(" + "|".join(RISK_LEVELS) + ")") + else: + pattern = re.compile( + r"(\d+)\.(\d+)-" + re.escape(flavor) + r"\/(" + "|".join(RISK_LEVELS) + ")" + ) + + # Dictionary to store the highest risk level for each major.minor + channel_map = {} + + for channel in available_channels: + match = pattern.match(channel) + if match: + major, minor, risk = match.groups() + major_minor = (int(major), int(minor)) + + # Store only the highest risk level channel for each major.minor + if major_minor not in channel_map or RISK_LEVELS.index( + risk + ) < RISK_LEVELS.index(channel_map[major_minor][1]): + channel_map[major_minor] = (channel, risk) + + # Sort channels by major and minor version in descending order + sorted_channels = sorted(channel_map.keys(), reverse=False) + + # Prepare final channel list + final_channels = [channel_map[mm][0] for mm in sorted_channels[:num_of_channels]] + + if include_latest: + latest_channel = f"latest/edge/{flavor}" + final_channels.append(latest_channel) + + return final_channels diff --git a/tests/integration/tests/test_version_upgrades.py b/tests/integration/tests/test_version_upgrades.py new file mode 100644 index 000000000..92ef2437e --- /dev/null +++ b/tests/integration/tests/test_version_upgrades.py @@ -0,0 +1,54 @@ +# +# Copyright 2024 Canonical, Ltd. +# +import logging +from typing import List + +import pytest +from test_util import config, harness, snap, util + +LOG = logging.getLogger(__name__) + + +@pytest.mark.node_count(1) +@pytest.mark.no_setup() +@pytest.mark.xfail("cilium failures are blocking this from working") +@pytest.mark.skipif( + not config.VERSION_UPGRADE_CHANNELS, reason="No upgrade channels configured" +) +def test_version_upgrades(instances: List[harness.Instance]): + channels = config.VERSION_UPGRADE_CHANNELS + cp = instances[0] + + if channels[0].lower() == "recent": + if len(channels) != 3: + pytest.fail( + "'recent' requires the number of releases as second argument and the flavour as third argument" + ) + _, num_channels, flavour = channels + arch = cp.exec( + ["dpkg", "--print-architecture"], text=True, capture_output=True + ).stdout.strip() + channels = snap.get_latest_channels(int(num_channels), flavour, arch) + + LOG.info( + f"Bootstrap node on {channels[0]} and upgrade through channels: {channels[1:]}" + ) + + # Setup the k8s snap from the bootstrap channel and setup basic configuration. + cp.exec(["snap", "install", "k8s", "--channel", channels[0], "--classic"]) + cp.exec(["k8s", "bootstrap"]) + + util.stubbornly(retries=30, delay_s=20).until(util.ready_nodes(cp) == 1) + + current_channel = channels[0] + for channel in channels[1:]: + LOG.info(f"Upgrading {cp.id} from {current_channel} to channel {channel}") + # Log the current snap version on the node. + cp.exec(["snap", "info", "k8s"]) + + # note: the `--classic` flag will be ignored by snapd for strict snaps. + cp.exec(["snap", "refresh", "k8s", "--channel", channel, "--classic"]) + + util.stubbornly(retries=30, delay_s=20).until(util.ready_nodes(cp) == 1) + LOG.info(f"Upgraded {cp.id} to channel {channel}")