From ade18f42c7eb5219ad049e6cda6709f266f6a4f2 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 26 Sep 2024 22:55:21 -0500 Subject: [PATCH 1/4] Use the correct snap when upgrading --- .github/workflows/nightly-test.yaml | 39 ++++++++++++++++++----------- tests/integration/tox.ini | 33 ++++++++++++------------ 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml index ea00bcd4f..6fd51df14 100644 --- a/.github/workflows/nightly-test.yaml +++ b/.github/workflows/nightly-test.yaml @@ -3,47 +3,47 @@ name: Nightly Latest/Edge Tests on: schedule: - cron: '0 0 * * *' # Runs every midnight + pull_request: permissions: contents: read jobs: test-integration: - name: Integration Test ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.releases }} + name: Integration Test ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.release }} strategy: matrix: os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"] arch: ["amd64", "arm64"] - releases: ["latest/edge"] + release: ["latest/edge"] fail-fast: false # TODO: remove once arm64 works - runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-20.04' }} + runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-22.04' }} steps: - name: Checking out repo uses: actions/checkout@v4 - - name: Setup Python + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + - name: Install lxd and tox run: | sudo apt update - sudo apt install -y python3 python3-pip - - name: Install tox - run: | - pip3 install tox==4.13 - - name: Install lxd - run: | + sudo apt install -y tox sudo snap refresh lxd --channel 5.21/stable sudo lxd init --auto sudo usermod --append --groups lxd $USER sg lxd -c 'lxc version' - name: Create build directory run: mkdir -p build - - name: Install $${ matrix.releases }} k8s snap + - name: Install ${{ matrix.release }} k8s snap run: | cd build - snap download k8s --channel=${{ matrix.releases }} --basename k8s + snap download k8s --channel=${{ matrix.release }} --basename k8s - name: Run end to end tests # tox path needs to be specified for arm64 env: - TEST_SNAP: ${{ github.workspace }}/build/k8s-${{ matrix.patch }}.snap + TEST_SNAP: ${{ github.workspace }}/build/k8s.snap TEST_SUBSTRATE: lxd TEST_LXD_IMAGE: ${{ matrix.os }} TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports @@ -52,4 +52,15 @@ jobs: TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic" run: | export PATH="/home/runner/.local/bin:$PATH" - cd tests/integration && sg lxd -c 'tox -e integration' + cd tests/integration && sg lxd -c 'tox -vve integration' + - name: Prepare inspection reports + if: failure() + run: | + tar -czvf inspection-reports.tar.gz -C ${{ github.workspace }} inspection-reports + echo "artifact_name=inspection-reports-${{ matrix.os }}-${{ matrix.arch }}" | sed 's/:/-/g' >> $GITHUB_ENV + - name: Upload inspection report artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ${{ env.artifact_name }} + path: ${{ github.workspace }}/inspection-reports.tar.gz diff --git a/tests/integration/tox.ini b/tests/integration/tox.ini index e2d7296b2..b59d696d7 100644 --- a/tests/integration/tox.ini +++ b/tests/integration/tox.ini @@ -1,47 +1,46 @@ [tox] -no_package = True +skipsdist = True skip_missing_interpreters = True env_list = format, lint, integration -min_version = 4.0.0 [testenv] set_env = PYTHONBREAKPOINT=pdb.set_trace PY_COLORS=1 -pass_env = +passenv = PYTHONPATH [testenv:format] description = Apply coding style standards to code -deps = -r {tox_root}/requirements-dev.txt +deps = -r {toxinidir}/requirements-dev.txt commands = - licenseheaders -t {tox_root}/.copyright.tmpl -cy -o 'Canonical, Ltd' -d {tox_root}/tests - isort {tox_root}/tests --profile=black - black {tox_root}/tests + licenseheaders -t {toxinidir}/.copyright.tmpl -cy -o 'Canonical, Ltd' -d {toxinidir}/tests + isort {toxinidir}/tests --profile=black + black {toxinidir}/tests [testenv:lint] description = Check code against coding style standards -deps = -r {tox_root}/requirements-dev.txt +deps = -r {toxinidir}/requirements-dev.txt commands = - codespell {tox_root}/tests - flake8 {tox_root}/tests - licenseheaders -t {tox_root}/.copyright.tmpl -cy -o 'Canonical, Ltd' -d {tox_root}/tests --dry - isort {tox_root}/tests --profile=black --check - black {tox_root}/tests --check --diff + codespell {toxinidir}/tests + flake8 {toxinidir}/tests + licenseheaders -t {toxinidir}/.copyright.tmpl -cy -o 'Canonical, Ltd' -d {toxinidir}/tests --dry + isort {toxinidir}/tests --profile=black --check + black {toxinidir}/tests --check --diff [testenv:integration] description = Run integration tests deps = - -r {tox_root}/requirements-test.txt + -r {toxinidir}/requirements-test.txt commands = - pytest -v \ + pytest -vv \ --maxfail 1 \ --tb native \ --log-cli-level DEBUG \ --disable-warnings \ {posargs} \ - {tox_root}/tests -pass_env = + {toxinidir}/tests +passenv = TEST_* [flake8] From 29c0ec01dea91e87674f93736ce953be9eb734fa Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Fri, 27 Sep 2024 08:10:14 -0500 Subject: [PATCH 2/4] Make etcd arch aware --- .github/workflows/nightly-test.yaml | 10 +++++----- tests/integration/templates/etcd/etcd.service | 1 + tests/integration/tests/test_util/etcd.py | 8 +++++--- tests/integration/tests/test_util/harness/base.py | 8 +++++++- tests/integration/tests/test_version_upgrades.py | 7 ++----- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml index 6fd51df14..b769bf434 100644 --- a/.github/workflows/nightly-test.yaml +++ b/.github/workflows/nightly-test.yaml @@ -18,15 +18,11 @@ jobs: release: ["latest/edge"] fail-fast: false # TODO: remove once arm64 works - runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-22.04' }} + runs-on: ${{ matrix.arch == 'arm64' && 'Ubuntu_ARM64_4C_16G_01' || 'ubuntu-20.04' }} steps: - name: Checking out repo uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - cache: 'pip' - name: Install lxd and tox run: | sudo apt update @@ -64,3 +60,7 @@ jobs: with: name: ${{ env.artifact_name }} path: ${{ github.workspace }}/inspection-reports.tar.gz + - name: Tmate debugging session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 10 diff --git a/tests/integration/templates/etcd/etcd.service b/tests/integration/templates/etcd/etcd.service index 0ac4e9933..e770666a0 100644 --- a/tests/integration/templates/etcd/etcd.service +++ b/tests/integration/templates/etcd/etcd.service @@ -11,6 +11,7 @@ LimitNOFILE=40000 TimeoutStartSec=0 + Environment=ETCD_UNSUPPORTED_ARCH=$ARCH ExecStart=/tmp/test-etcd/etcd --name $NAME \ --data-dir /tmp/etcd/s1 \ --listen-client-urls $CLIENT_URL \ diff --git a/tests/integration/tests/test_util/etcd.py b/tests/integration/tests/test_util/etcd.py index 44f8a0eee..d53e8ee66 100644 --- a/tests/integration/tests/test_util/etcd.py +++ b/tests/integration/tests/test_util/etcd.py @@ -90,6 +90,7 @@ def add_node(self): ] substitutes = { + "ARCH": instance.arch, "NAME": instance.id, "IP": ip, "CLIENT_URL": f"https://{ip}:2379", @@ -224,13 +225,14 @@ def add_node(self): input=str.encode(src.substitute(substitutes)), ) + arch = instance.arch instance.exec( [ "curl", "-L", - f"{self.etcd_url}/{self.etcd_version}/etcd-{self.etcd_version}-linux-amd64.tar.gz", + f"{self.etcd_url}/{self.etcd_version}/etcd-{self.etcd_version}-linux-{arch}.tar.gz", "-o", - f"/tmp/etcd-{self.etcd_version}-linux-amd64.tar.gz", + f"/tmp/etcd-{self.etcd_version}-linux-{arch}.tar.gz", ] ) instance.exec(["mkdir", "-p", "/tmp/test-etcd"]) @@ -238,7 +240,7 @@ def add_node(self): [ "tar", "xzvf", - f"/tmp/etcd-{self.etcd_version}-linux-amd64.tar.gz", + f"/tmp/etcd-{self.etcd_version}-linux-{arch}.tar.gz", "-C", "/tmp/test-etcd", "--strip-components=1", diff --git a/tests/integration/tests/test_util/harness/base.py b/tests/integration/tests/test_util/harness/base.py index 829d64511..8111a339d 100644 --- a/tests/integration/tests/test_util/harness/base.py +++ b/tests/integration/tests/test_util/harness/base.py @@ -2,7 +2,7 @@ # Copyright 2024 Canonical, Ltd. # import subprocess -from functools import partial +from functools import cached_property, partial class HarnessError(Exception): @@ -30,6 +30,12 @@ def __init__(self, h: "Harness", id: str) -> None: def id(self) -> str: return self._id + @cached_property + def arch(self) -> str: + return self.exec( + ["dpkg", "--print-architecture"], text=True, capture_output=True + ).stdout.strip() + def __str__(self) -> str: return f"{self._h.name}:{self.id}" diff --git a/tests/integration/tests/test_version_upgrades.py b/tests/integration/tests/test_version_upgrades.py index 92ef2437e..d6e7a45b0 100644 --- a/tests/integration/tests/test_version_upgrades.py +++ b/tests/integration/tests/test_version_upgrades.py @@ -12,7 +12,7 @@ @pytest.mark.node_count(1) @pytest.mark.no_setup() -@pytest.mark.xfail("cilium failures are blocking this from working") +@pytest.mark.xfail(reason="cilium failures are blocking this from working") @pytest.mark.skipif( not config.VERSION_UPGRADE_CHANNELS, reason="No upgrade channels configured" ) @@ -26,10 +26,7 @@ def test_version_upgrades(instances: List[harness.Instance]): "'recent' requires the number of releases as second argument and the flavour as third argument" ) _, num_channels, flavour = channels - arch = cp.exec( - ["dpkg", "--print-architecture"], text=True, capture_output=True - ).stdout.strip() - channels = snap.get_latest_channels(int(num_channels), flavour, arch) + channels = snap.get_latest_channels(int(num_channels), flavour, cp.arch) LOG.info( f"Bootstrap node on {channels[0]} and upgrade through channels: {channels[1:]}" From 8af94cc8d50ae8d3e51ed9a17543fb12d93001d7 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Fri, 27 Sep 2024 11:19:24 -0500 Subject: [PATCH 3/4] bump etcd version to 3.4.34 --- tests/integration/tests/test_util/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/tests/test_util/config.py b/tests/integration/tests/test_util/config.py index b9e535683..fddef4e2a 100644 --- a/tests/integration/tests/test_util/config.py +++ b/tests/integration/tests/test_util/config.py @@ -15,7 +15,7 @@ ETCD_URL = os.getenv("ETCD_URL") or "https://github.com/etcd-io/etcd/releases/download" # ETCD_VERSION is the version of etcd to use. -ETCD_VERSION = os.getenv("ETCD_VERSION") or "v3.3.8" +ETCD_VERSION = os.getenv("ETCD_VERSION") or "v3.4.34" # SNAP is the absolute path to the snap against which we run the integration tests. SNAP = os.getenv("TEST_SNAP") From 86176e8f706111985a64d5242df9d21ec02da819 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Fri, 27 Sep 2024 14:03:34 -0500 Subject: [PATCH 4/4] Only use tmate if we're debugging a pull-reqeust --- .github/workflows/nightly-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml index b769bf434..c198dd8dd 100644 --- a/.github/workflows/nightly-test.yaml +++ b/.github/workflows/nightly-test.yaml @@ -61,6 +61,6 @@ jobs: name: ${{ env.artifact_name }} path: ${{ github.workspace }}/inspection-reports.tar.gz - name: Tmate debugging session - if: ${{ failure() }} + if: ${{ failure() && github.event_name == 'pull_request' }} uses: mxschmitt/action-tmate@v3 timeout-minutes: 10