From c061487dc9574c9a12faeba386d3cc25a6329019 Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Thu, 23 Mar 2023 11:11:30 +0000 Subject: [PATCH] Remove Iris' TestRunner (#5205) --- .github/workflows/ci-tests.yml | 39 +++-- .github/workflows/ci-wheels.yml | 18 +-- .../{nox-cache => tox-cache}/action.yml | 10 +- docs/src/whatsnew/latest.rst | 3 + lib/iris/config.py | 3 +- lib/iris/tests/runner/__init__.py | 9 -- lib/iris/tests/runner/__main__.py | 39 ----- lib/iris/tests/runner/_runner.py | 143 ------------------ noxfile.py | 18 +-- tox.ini | 37 +++++ 10 files changed, 86 insertions(+), 233 deletions(-) rename .github/workflows/composite/{nox-cache => tox-cache}/action.yml (65%) delete mode 100644 lib/iris/tests/runner/__init__.py delete mode 100644 lib/iris/tests/runner/__main__.py delete mode 100644 lib/iris/tests/runner/_runner.py create mode 100644 tox.ini diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 46cc319c49b..f00671b857c 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -35,22 +35,31 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: ["3.10"] - session: ["doctest", "gallery", "linkcheck"] + python-version: ["310"] + session: ["docs-tests", "docs-linkcheck"] include: - os: "ubuntu-latest" - python-version: "3.10" + python-version: "310" session: "tests" - coverage: "--coverage" + posargs: "--gallery-tests" - os: "ubuntu-latest" - python-version: "3.9" + python-version: "310" session: "tests" + posargs: "--coverage" - os: "ubuntu-latest" - python-version: "3.8" + python-version: "39" session: "tests" + - os: "ubuntu-latest" + python-version: "38" + session: "tests" + - os: "ubuntu-latest" + python-version: "310" + session: "tests" + posargs: "--gallery-tests" + env: - IRIS_TEST_DATA_VERSION: "2.19" + IRIS_TEST_DATA_VERSION: "2.18" ENV_NAME: "ci-tests" steps: @@ -63,7 +72,7 @@ jobs: CACHE_WEEKS: 2 run: | echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV} - echo "LOCK_FILE=requirements/ci/nox.lock/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV} + echo "LOCK_FILE=requirements/ci/locks/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV} - name: "data cache" uses: ./.github/workflows/composite/iris-data-cache @@ -91,10 +100,10 @@ jobs: - name: "conda environment cache" uses: ./.github/workflows/composite/conda-env-cache with: - cache_build: 0 + cache_build: 1 cache_period: ${{ env.CACHE_PERIOD }} env_name: ${{ env.ENV_NAME }} - install_packages: "cartopy nox pip" + install_packages: "cartopy tox<4" - name: "conda info" run: | @@ -108,8 +117,8 @@ jobs: cache_period: ${{ env.CACHE_PERIOD }} env_name: ${{ env.ENV_NAME }} - - name: "nox cache" - uses: ./.github/workflows/composite/nox-cache + - name: "tox cache" + uses: ./.github/workflows/composite/tox-cache with: cache_build: 0 env_name: ${{ env.ENV_NAME }} @@ -134,11 +143,9 @@ jobs: cat ${MPL_RC} - name: "iris ${{ matrix.session }}" - env: - PY_VER: ${{ matrix.python-version }} run: | - nox --session ${{ matrix.session }} -- --verbose ${{ matrix.coverage }} + tox -e py${{ matrix.python-version }}-${{ matrix.session }} -- ${{ matrix.posargs }} - name: Upload coverage report uses: codecov/codecov-action@v3 - if: ${{ matrix.coverage }} + if: ${{ matrix.posargs }} == "--coverage" diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index a00833b1182..621184d087b 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -54,7 +54,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["38", "39", "310"] session: ["wheel"] env: ENV_NAME: "ci-wheels" @@ -74,7 +74,7 @@ jobs: CACHE_WEEKS: 2 run: | echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV} - echo "LOCK_FILE=requirements/ci/nox.lock/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV} + echo "LOCK_FILE=requirements/ci/locks/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV} - name: "conda package cache" uses: ./.github/workflows/composite/conda-pkg-cache @@ -95,23 +95,21 @@ jobs: - name: "conda environment cache" uses: ./.github/workflows/composite/conda-env-cache with: - cache_build: 0 + cache_build: 1 cache_period: ${{ env.CACHE_PERIOD }} env_name: ${{ env.ENV_NAME }} - install_packages: "nox pip" + install_packages: "tox<4 pip" - - name: "nox cache" - uses: ./.github/workflows/composite/nox-cache + - name: "tox cache" + uses: ./.github/workflows/composite/tox-cache with: cache_build: 0 env_name: ${{ env.ENV_NAME }} lock_file: ${{ env.LOCK_FILE }} - - name: "nox install and test wheel" - env: - PY_VER: ${{ matrix.python-version }} + - name: "tox install and test wheel" run: | - nox --session ${{ matrix.session }} -- --verbose + tox -e py${{ matrix.python-version }}-${{ matrix.session }} show-artifacts: needs: build diff --git a/.github/workflows/composite/nox-cache/action.yml b/.github/workflows/composite/tox-cache/action.yml similarity index 65% rename from .github/workflows/composite/nox-cache/action.yml rename to .github/workflows/composite/tox-cache/action.yml index 468dd22d81b..48962f9d51d 100644 --- a/.github/workflows/composite/nox-cache/action.yml +++ b/.github/workflows/composite/tox-cache/action.yml @@ -1,9 +1,9 @@ -name: "nox cache" -description: "cache the nox test environments" +name: "tox cache" +description: "cache the tox test environments" inputs: cache_build: - description: "nox cache build number" + description: "tox cache build number" required: false default: "0" env_name: @@ -18,5 +18,5 @@ runs: steps: - uses: actions/cache@v3 with: - path: ${{ github.workspace }}/.nox - key: ${{ runner.os }}-nox-${{ inputs.env_name }}-s${{ matrix.session }}-py${{ matrix.python-version }}-b${{ inputs.cache_build }}-${{ hashFiles(inputs.lock_file) }} + path: ${{ github.workspace }}/.tox + key: ${{ runner.os }}-tox-${{ inputs.env_name }}-s${{ matrix.session }}-py${{ matrix.python-version }}-b${{ inputs.cache_build }}-${{ hashFiles(inputs.lock_file) }} diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index d92711d2858..d319a32ca40 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -137,6 +137,9 @@ This document explains the changes made to Iris for this release #. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) removed the ``--coding-tests`` option from Iris' test runner. (:pull:`4765`) +#. `@lbdreyer`_ removed the Iris TestRunner. Tests are now run via nox or + pytest. (:pull:`5205`) + .. comment Whatsnew author names (@github name) in alphabetical order. Note that, diff --git a/lib/iris/config.py b/lib/iris/config.py index 3659ac7dcd6..79d141e53f3 100644 --- a/lib/iris/config.py +++ b/lib/iris/config.py @@ -171,8 +171,7 @@ def get_dir_option(section, option, default=None): ) # Override the data repository if the appropriate environment variable -# has been set. This is used in setup.py in the TestRunner command to -# enable us to simulate the absence of external data. +# has been set. override = os.environ.get("OVERRIDE_TEST_DATA_REPOSITORY") if override: TEST_DATA_DIR = None diff --git a/lib/iris/tests/runner/__init__.py b/lib/iris/tests/runner/__init__.py deleted file mode 100644 index b561e1cf875..00000000000 --- a/lib/iris/tests/runner/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Iris contributors -# -# This file is part of Iris and is released under the LGPL license. -# See COPYING and COPYING.LESSER in the root of the repository for full -# licensing details. -""" -Empty file to allow import. - -""" diff --git a/lib/iris/tests/runner/__main__.py b/lib/iris/tests/runner/__main__.py deleted file mode 100644 index 9f9c51c1f7d..00000000000 --- a/lib/iris/tests/runner/__main__.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright Iris contributors -# -# This file is part of Iris and is released under the LGPL license. -# See COPYING and COPYING.LESSER in the root of the repository for full -# licensing details. -""" -Provides testing capabilities for installed copies of Iris. - -""" - -import argparse - -from ._runner import TestRunner - -parser = argparse.ArgumentParser( - "iris.tests", description=TestRunner.description -) -for long_opt, short_opt, help_text in TestRunner.user_options: - long_opt = long_opt.strip("=") - if long_opt in TestRunner.boolean_options: - parser.add_argument( - "--" + long_opt, - "-" + short_opt, - action="store_true", - help=help_text, - ) - else: - parser.add_argument("--" + long_opt, "-" + short_opt, help=help_text) -args = parser.parse_args() - -runner = TestRunner() - -runner.initialize_options() -for long_opt, short_opt, help_text in TestRunner.user_options: - arg = long_opt.replace("-", "_").strip("=") - setattr(runner, arg, getattr(args, arg)) -runner.finalize_options() - -runner.run() diff --git a/lib/iris/tests/runner/_runner.py b/lib/iris/tests/runner/_runner.py deleted file mode 100644 index 7f9439d4b61..00000000000 --- a/lib/iris/tests/runner/_runner.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright Iris contributors -# -# This file is part of Iris and is released under the LGPL license. -# See COPYING and COPYING.LESSER in the root of the repository for full -# licensing details. -""" -Provides testing capabilities for installed copies of Iris. - -""" - -# Because this file is imported by setup.py, there may be additional runtime -# imports later in the file. -import os -import sys - - -# NOTE: Do not inherit from object as distutils does not like it. -class TestRunner: - """Run the Iris tests under pytest and pytest-xdist for performance""" - - description = ( - "Run tests under pytest and pytest-xdist for performance. " - "Default behaviour is to run all non-gallery tests. " - "Specifying one or more test flags will run *only* those " - "tests." - ) - user_options = [ - ( - "no-data", - "n", - "Override the paths to the data repositories so it " - "appears to the tests that it does not exist.", - ), - ("stop", "x", "Stop running tests after the first error or failure."), - ("system-tests", "s", "Run the limited subset of system tests."), - ("gallery-tests", "e", "Run the gallery code tests."), - ("default-tests", "d", "Run the default tests."), - ( - "num-processors=", - "p", - "The number of processors used for running " "the tests.", - ), - ("create-missing", "m", "Create missing test result files."), - ("coverage", "c", "Enable coverage testing"), - ] - boolean_options = [ - "no-data", - "system-tests", - "stop", - "gallery-tests", - "default-tests", - "create-missing", - "coverage", - ] - - def initialize_options(self): - self.no_data = False - self.stop = False - self.system_tests = False - self.gallery_tests = False - self.default_tests = False - self.num_processors = None - self.create_missing = False - self.coverage = False - - def finalize_options(self): - # These environment variables will be propagated to all the - # processes that pytest-xdist creates. - if self.no_data: - print("Running tests in no-data mode...") - import iris.config - - iris.config.TEST_DATA_DIR = None - if self.create_missing: - os.environ["IRIS_TEST_CREATE_MISSING"] = "true" - - tests = [] - if self.system_tests: - tests.append("system") - if self.default_tests: - tests.append("default") - if self.gallery_tests: - tests.append("gallery") - if not tests: - tests.append("default") - print("Running test suite(s): {}".format(", ".join(tests))) - if self.stop: - print("Stopping tests after the first error or failure") - if self.num_processors is None: - self.num_processors = "auto" - else: - self.num_processors = int(self.num_processors) - - def run(self): - import pytest - - if hasattr(self, "distribution") and self.distribution.tests_require: - self.distribution.fetch_build_eggs(self.distribution.tests_require) - - tests = [] - if self.system_tests: - tests.append("lib/iris/tests/system_test.py") - if self.default_tests: - tests.append("lib/iris/tests") - if self.gallery_tests: - import iris.config - - default_doc_path = os.path.join(sys.path[0], "docs") - doc_path = iris.config.get_option( - "Resources", "doc_dir", default=default_doc_path - ) - gallery_path = os.path.join(doc_path, "gallery_tests") - if os.path.exists(gallery_path): - tests.append(gallery_path) - else: - print( - "WARNING: Gallery path %s does not exist." % (gallery_path) - ) - if not tests: - tests.append("lib/iris/tests") - - args = [ - None, - f"-n={self.num_processors}", - ] - - if self.stop: - args.append("-x") - - if self.coverage: - args.extend(["--cov=lib/iris", "--cov-report=xml"]) - - result = True - for test in tests: - args[0] = test - print() - print( - f"Running test discovery on {test} with {self.num_processors} processors." - ) - retcode = pytest.main(args=args) - result &= retcode.value == 0 - if result is False: - exit(1) diff --git a/noxfile.py b/noxfile.py index f02b91f8236..f3210cfc1b2 100755 --- a/noxfile.py +++ b/noxfile.py @@ -188,13 +188,13 @@ def tests(session: nox.sessions.Session): session.install("--no-deps", "--editable", ".") session.env.update(ENV) run_args = [ - "python", - "-m", - "iris.tests.runner", - "--default-tests", + "pytest", + "-n", + "auto", + "lib/iris/tests", ] if "-c" in session.posargs or "--coverage" in session.posargs: - run_args.append("--coverage") + run_args[-1:-1] = ["--cov=lib/iris", "--cov-report=xml"] session.run(*run_args) @@ -241,10 +241,10 @@ def gallery(session: nox.sessions.Session): session.install("--no-deps", "--editable", ".") session.env.update(ENV) session.run( - "python", - "-m", - "iris.tests.runner", - "--gallery-tests", + "pytest", + "-n", + "auto", + "docs/gallery_tests", ) diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000000..1830e7805fb --- /dev/null +++ b/tox.ini @@ -0,0 +1,37 @@ +[tox] +requires = + tox-conda + +[testenv] +conda_spec = + py38: {toxinidir}{/}requirements{/}locks{/}py38-linux-64.lock + py39: {toxinidir}{/}requirements{/}locks{/}py39-linux-64.lock + py310: {toxinidir}{/}requirements{/}locks{/}py310-linux-64.lock + +[testenv:py{38,39,310}-tests] +description = Perform Iris unit, integration and system tests. +usedevelop = + true +commands = + python -m iris.tests.runner {posargs} + +[testenv:py{38,39,310}-docs{,-linkcheck,-tests}] +description = Build, test and link-check Iris documentation +allowlist_externals = + make +changedir = + {toxinidir}{/}docs +usedevelop = + true +commands = + make clean html + linkcheck: make linkcheck + tests: make doctest + +[testenv:py{38,39,310}-wheel] +description = Install wheel and test Iris import +skip_install = true +deps = + {toxinidir}/dist/scitools_iris*.whl +commands = + python -c "import iris; print(f'\{iris.__version__=\}')" \ No newline at end of file