diff --git a/docs/config.rst b/docs/config.rst index 728bcb4fea..5c3f1bdc93 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -967,7 +967,4 @@ Other Rules and notes cli === -.. autoprogram:: tox.cli:cli - :prog: tox - .. include:: links.rst diff --git a/docs/plugins.rst b/docs/plugins.rst index e6e714a3d3..61b045fc5a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -187,22 +187,11 @@ You can and publish it like: Hook specifications and related API ----------------------------------- -.. automodule:: tox.hookspecs +.. automodule:: tox.plugin.spec :members: -.. autoclass:: tox.config.Parser() +.. autoclass:: tox.config.sets.ConfigSet :members: -.. autoclass:: tox.config.Config() - :members: - -.. autoclass:: tox.config.TestenvConfig() - :members: - -.. autoclass:: tox.venv.VirtualEnv() - :members: - -.. autoclass:: tox.session.Session() - :members: .. include:: links.rst diff --git a/setup.cfg b/setup.cfg index 1c36b10fd6..4625d97199 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ classifiers = [options] packages = find: -python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +python_requires = >=3.5 install_requires = importlib-metadata >= 0.12, <1;python_version<"3.8" packaging >= 14 diff --git a/src/tox/__init__.py b/src/tox/__init__.py index e69de29bb2..dc2782c3f6 100644 --- a/src/tox/__init__.py +++ b/src/tox/__init__.py @@ -0,0 +1,3 @@ +from .version import __version__ + +__all__ = ("__version__",) diff --git a/src/tox/config/cli/for_docs.py b/src/tox/config/cli/for_docs.py new file mode 100644 index 0000000000..95f6393fa2 --- /dev/null +++ b/src/tox/config/cli/for_docs.py @@ -0,0 +1,3 @@ +from .parse import _get_core_parser + +parser = _get_core_parser() diff --git a/src/tox/config/cli/parse.py b/src/tox/config/cli/parse.py index 8ecb4977e4..07fbb45407 100644 --- a/src/tox/config/cli/parse.py +++ b/src/tox/config/cli/parse.py @@ -22,12 +22,16 @@ def _get_base(args): def _get_core(args): + tox_parser = _get_core_parser() + parsed, unknown = tox_parser.parse(args) + handlers = {k: p for k, (_, p) in tox_parser.handlers.items()} + return handlers, parsed, unknown + + +def _get_core_parser(): tox_parser = ToxParser.core() # noinspection PyUnresolvedReferences from tox.plugin.manager import MANAGER - MANAGER.tox_add_option(tox_parser) tox_parser.fix_defaults() - parsed, unknown = tox_parser.parse(args) - handlers = {k: p for k, (_, p) in tox_parser.handlers.items()} - return handlers, parsed, unknown + return tox_parser diff --git a/src/tox/config/main.py b/src/tox/config/main.py index 12ad121ff7..ec3fa90a80 100644 --- a/src/tox/config/main.py +++ b/src/tox/config/main.py @@ -38,3 +38,6 @@ def __iter__(self): def __repr__(self): return "{}(config_source={!r})".format(type(self).__name__, self._src) + + def __contains__(self, item): + return item in self._envs diff --git a/src/tox/config/sets.py b/src/tox/config/sets.py index 21b8f079a7..e1beb1f6d5 100644 --- a/src/tox/config/sets.py +++ b/src/tox/config/sets.py @@ -117,6 +117,17 @@ def add_config( post_process=None, overwrite=False, ): + """ + Add configuration. + + :param keys: + :param of_type: + :param default: + :param desc: + :param post_process: + :param overwrite: + :return: + """ keys_ = self._make_keys(keys) for key in keys_: if key in self._defined and overwrite is False: diff --git a/src/tox/config/source/ini/__init__.py b/src/tox/config/source/ini/__init__.py index d95a4a9173..5c5369ba57 100644 --- a/src/tox/config/source/ini/__init__.py +++ b/src/tox/config/source/ini/__init__.py @@ -9,9 +9,8 @@ from ..api import EnvList, Loader, Source from .convert import StrConvert from .factor import filter_for_env, find_envs -from .replace import replace +from .replace import BASE_TEST_ENV, replace -BASE_TEST_ENV = "testenv" TEST_ENV_PREFIX = "{}:".format(BASE_TEST_ENV) @@ -28,6 +27,7 @@ def __init__(self, path: Path) -> None: src=self, name=None, default_base=EnvList([]), + section_loader=self._get_section, ) super().__init__(core) self._envs = {} # type: Dict[str, IniLoader] @@ -70,7 +70,9 @@ def get_section(self, item, name): try: return self._envs[item] except KeyError: - loader = IniLoader(self._get_section(item), self, name, self.BASE_ENV_LIST) + loader = IniLoader( + self._get_section(item), self, name, self.BASE_ENV_LIST, self._get_section + ) self._envs[item] = loader return loader @@ -104,13 +106,19 @@ class IniLoader(Loader, StrConvert): """Load from a ini section""" def __init__( - self, section: Optional[SectionProxy], src: Ini, name: Optional[str], default_base: EnvList + self, + section: Optional[SectionProxy], + src: Ini, + name: Optional[str], + default_base: EnvList, + section_loader, ) -> None: super().__init__(name) self._section = section # type:Optional[SectionProxy] self._src = src # type: Ini self._default_base = default_base # type:EnvList self._base = [] # type:List[IniLoader] + self._section_loader = section_loader def __deepcopy__(self, memo): # python < 3.7 cannot copy config parser @@ -127,10 +135,11 @@ def __deepcopy__(self, memo): def setup_with_conf(self, conf: ConfigSet): # noinspection PyUnusedLocal def load_bases(values, conf_): - result = [] + result = [] # type: List[IniLoader] for value in values: name = value.lstrip(TEST_ENV_PREFIX) - result.append(self._src.get_section(value, name)) + ini_loader = self._src.get_section(value, name) # type: IniLoader + result.append(ini_loader) return result conf.add_config( @@ -169,11 +178,16 @@ def _load_raw_from(self, as_name, conf, key): raise KeyError(key) value = self._section[key] collapsed_newlines = value.replace("\\\n", "") # collapse explicit line splits - replace_executed = replace(collapsed_newlines, conf, as_name) # do replacements + replace_executed = replace( + collapsed_newlines, conf, as_name, self._section_loader + ) # do replacements factor_selected = filter_for_env(replace_executed, as_name) # select matching factors # extend factors return factor_selected + def get_value(self, section, key): + return self._section_loader(section)[key] + @property def loaders(self): yield self diff --git a/src/tox/config/source/ini/replace.py b/src/tox/config/source/ini/replace.py index ad5d849d02..07bd1bd09b 100644 --- a/src/tox/config/source/ini/replace.py +++ b/src/tox/config/source/ini/replace.py @@ -16,23 +16,24 @@ """, re.VERBOSE, ) +BASE_TEST_ENV = "testenv" -def substitute_once(val, conf, name): +def substitute_once(val, conf, name, section_loader): # noinspection PyTypeChecker - return RE_ITEM_REF.sub(partial(_replace_match, conf, name), val) + return RE_ITEM_REF.sub(partial(_replace_match, conf, name, section_loader), val) -def replace(value, conf, name): +def replace(value, conf, name, section_loader): while True: # substitution found - expanded = substitute_once(value, conf, name) + expanded = substitute_once(value, conf, name, section_loader) if expanded == value: break value = expanded return expanded -def _replace_match(conf: Config, name, match): +def _replace_match(conf: Config, name, section_loader, match): groups = match.groupdict() sub_type = groups["sub_type"] value = groups["substitution_value"] @@ -50,10 +51,16 @@ def _replace_match(conf: Config, name, match): if sub_type is not None: key_missing_value = value value = sub_type + else: + value = groups["key"] section = groups["section"] or name # noinspection PyBroadException + if section not in conf: + env_conf = section_loader(section) + else: + env_conf = conf[section] try: - replace_value = conf[section][value] + replace_value = env_conf[value] except Exception: # noinspection PyBroadException try: diff --git a/src/tox/session/cmd/list_env.py b/src/tox/session/cmd/list_env.py index c524ad06ff..f200783ddb 100644 --- a/src/tox/session/cmd/list_env.py +++ b/src/tox/session/cmd/list_env.py @@ -24,7 +24,7 @@ def list_env(state: State): if not option.list_quiet and default: print("default environments:") - max_length = max(len(env) for env in (default + extra)) + max_length = max(len(env) for env in (default.envs + extra)) def report_env(name: str): if not option.list_quiet: diff --git a/src/tox/session/cmd/run/single.py b/src/tox/session/cmd/run/single.py index d90edb2c79..8e96c13444 100644 --- a/src/tox/session/cmd/run/single.py +++ b/src/tox/session/cmd/run/single.py @@ -13,7 +13,7 @@ def run_one(tox_env: RunToxEnv, recreate: bool, no_test: bool) -> int: try: tox_env.setup() except Recreate: - tox_env.clean(package_env=recreate) + tox_env.clean(package_env=False) # restart creation once, no package please tox_env.setup() code = run_commands(tox_env, no_test) diff --git a/src/tox/tox_env/package.py b/src/tox/tox_env/package.py index 3900172331..3906e86759 100644 --- a/src/tox/tox_env/package.py +++ b/src/tox/tox_env/package.py @@ -1,10 +1,19 @@ from abc import ABC, abstractmethod from typing import Any, List +from tox.config.sets import ConfigSet +from tox.execute.api import Execute +from tox.tox_env.errors import Recreate + from .api import ToxEnv class PackageToxEnv(ToxEnv, ABC): + def __init__(self, conf: ConfigSet, core: ConfigSet, options, executor: Execute): + super().__init__(conf, core, options, executor) + self._cleaned = False + self._setup_done = False + def register_config(self): super().register_config() @@ -15,3 +24,17 @@ def get_package_dependencies(self, extras=None) -> List[Any]: @abstractmethod def perform_packaging(self) -> List[Any]: raise NotImplementedError + + def clean(self): + # package environments may be shared clean only once + if self._cleaned is False: + self._cleaned = True + super().clean() + + def ensure_setup(self): + if self._setup_done is False: + try: + self.setup() + except Recreate: + self.clean() + self.setup() diff --git a/src/tox/tox_env/python/api.py b/src/tox/tox_env/python/api.py index ef70717bd7..4d0284b737 100644 --- a/src/tox/tox_env/python/api.py +++ b/src/tox/tox_env/python/api.py @@ -93,7 +93,8 @@ def cached_install(self, deps, section, of_type): if missing: # no way yet to know what to uninstall here (transitive dependencies?) # bail out and force recreate raise Recreate() - new_deps = [Requirement(i) for i in (set(conf_deps) - set(old))] + new_deps_str = set(conf_deps) - set(old) + new_deps = [Requirement(i) for i in new_deps_str] self.install_python_packages(packages=new_deps) @abstractmethod diff --git a/src/tox/tox_env/python/runner.py b/src/tox/tox_env/python/runner.py index 706f718b4a..a5994eb1a5 100644 --- a/src/tox/tox_env/python/runner.py +++ b/src/tox/tox_env/python/runner.py @@ -3,7 +3,7 @@ from packaging.requirements import Requirement -from tox.tox_env.errors import Recreate, Skip +from tox.tox_env.errors import Skip from ..runner import RunToxEnv from .api import NoInterpreter, Python @@ -49,12 +49,6 @@ def setup(self) -> None: self.cached_install(self.conf["deps"], PythonRun.__name__, "deps") if self.package_env is not None: - try: - self.package_env.setup() - except Recreate: - self.package_env.clean() - self.package_env.setup() - package_deps = self.package_env.get_package_dependencies(self.conf["extras"]) self.cached_install(package_deps, PythonRun.__name__, "package_deps") self.install_package() diff --git a/src/tox/tox_env/python/virtual_env/package/api.py b/src/tox/tox_env/python/virtual_env/package/api.py index 7cf3476e89..0e7516aad0 100644 --- a/src/tox/tox_env/python/virtual_env/package/api.py +++ b/src/tox/tox_env/python/virtual_env/package/api.py @@ -106,8 +106,8 @@ def get_package_dependencies(self, extras=None) -> List[Requirement]: if key == "Requires-Dist": req = Requirement(v) markers = getattr(req.marker, "_markers", tuple()) or tuple() - for _at, (m_key, op, m_val) in enumerate( - i for i in markers if isinstance(i, tuple) and len(i) == 3 + for _at, (m_key, op, m_val) in ( + (j, i) for j, i in enumerate(markers) if isinstance(i, tuple) and len(i) == 3 ): if m_key.value == "extra" and op.value == "==": extra = m_val.value @@ -117,15 +117,19 @@ def get_package_dependencies(self, extras=None) -> List[Requirement]: if extra is None or extra in extras: if _at is not None: # noinspection PyProtectedMember - del req.marker._markers[_at] + del markers[_at] + _at -= 1 + if _at > 0 and markers[_at] in ("and", "or"): + del markers[_at] # noinspection PyProtectedMember - if len(req.marker._markers) == 0: + if len(markers) == 0: req.marker = None result.append(req) return result def _ensure_meta_present(self): if self._distribution_meta is None: + self.ensure_setup() self.meta_folder.mkdir(exist_ok=True) cmd = [ "python", diff --git a/src/tox/tox_env/python/virtual_env/package/artifact/api.py b/src/tox/tox_env/python/virtual_env/package/artifact/api.py index 25459e3350..41316eac17 100644 --- a/src/tox/tox_env/python/virtual_env/package/artifact/api.py +++ b/src/tox/tox_env/python/virtual_env/package/artifact/api.py @@ -48,5 +48,6 @@ def _build_artifact(self) -> List[Path]: def perform_packaging(self) -> List[Path]: """build_wheel/build_sdist""" if self._package is None: + self.ensure_setup() self._package = self._build_artifact() return self._package diff --git a/tests/unit/config/ini/replace/test_replace_tox_env.py b/tests/unit/config/ini/replace/test_replace_tox_env.py index ee14bd47f8..e45b78308a 100644 --- a/tests/unit/config/ini/replace/test_replace_tox_env.py +++ b/tests/unit/config/ini/replace/test_replace_tox_env.py @@ -54,3 +54,11 @@ def test_replace_within_tox_env_missing_default_env_only(example): env_config.add_config(keys="o", of_type=str, default="o", desc="o") result = env_config["o"] assert result == "one" + + +def test_replace_within_tox_env_from_base(example): + env_config = example("p = one\n[testenv:a]\no = {[testenv]p}") + env_config.add_config(keys="p", of_type=str, default="p", desc="p") + env_config.add_config(keys="o", of_type=str, default="o", desc="o") + result = env_config["o"] + assert result == "one" diff --git a/tox.ini b/tox.ini index cc5cb76f8b..377645e11f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] envlist = py38 - py37 py35 + py37 py36 - pypy3 coverage fix_lint docs @@ -16,7 +15,7 @@ requires = virtualenv >= 10 [testenv] -description = run the tests with pytest under {basepython} +description = run the tests with pytest setenv = COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}} VIRTUALENV_NO_DOWNLOAD = 1 passenv = http_proxy https_proxy no_proxy SSL_CERT_FILE PYTEST_* @@ -40,7 +39,7 @@ description = invoke sphinx-build to build the HTML docs basepython = python3.7 extras = docs commands = sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out" --color -W -bhtml {posargs} - python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' + python -c 'import pathlib; print("documentation available under file://{0}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' [testenv:package_description] description = check that the long description is valid @@ -49,7 +48,7 @@ deps = twine >= 1.12.1 # TODO installing readme-renderer[md] should not be necessary readme-renderer[md] >= 24.0 pip >= 18.0.0 -skip_install = true +package = skip extras = commands = pip wheel -w {envtmpdir}/build --no-deps . twine check {envtmpdir}/build/* @@ -64,7 +63,7 @@ passenv = {[testenv]passenv} extras = lint deps = pre-commit >= 1.14.4, < 2 -skip_install = True +package = skip commands = pre-commit run --all-files --show-diff-on-failure {posargs} python -c 'import pathlib; print("hint: run {} install to add checks as pre-commit hook".format(pathlib.Path(r"{envdir}") / "bin" / "pre-commit"))' @@ -75,7 +74,7 @@ description = [run locally after tests]: combine coverage data and create report deps = {[testenv]deps} coverage >= 4.4.1, < 5 diff_cover -skip_install = True +package = skip passenv = {[testenv]passenv} DIFF_AGAINST setenv = COVERAGE_FILE={toxworkdir}/.coverage @@ -92,7 +91,7 @@ parallel_show_output = True # PYTHONPATH=.:$PYTHONPATH python3 -m tox -e exit_code basepython = python3.7 description = commands with several exit codes -skip_install = True +package = skip commands = python3.7 -c "import sys; sys.exit(139)" [testenv:X] @@ -160,7 +159,7 @@ commands = python {toxinidir}/tasks/release.py --version {posargs} [testenv:notify] description = notify people about the release of the library basepython = python3.7 -skip_install = true +package = skip passenv = * deps = gitpython >= 2.1.10 packaging >= 17.1