python 3.5 friendly

azure-pipelines.yml

@@ -33,23 +33,17 @@ jobs:
         image: [linux]
       py37:
         image: [linux, windows, macOs]
-      py27:
-        image: [linux, windows, macOs]
-      pypy:
-        image: [linux]
       pypy3:
         image: [linux]
       py36:
         image: [linux, windows, macOs]
       py35:
         image: [linux, windows, macOs]
-      py34:
-        image: [linux, windows, macOs]
       dev: null
       package_description: null
     coverage:
       with_toxenv: 'coverage' # generate .tox/.coverage, .tox/coverage.xml after test run
-      for_envs: [py37, py36, py35, py34, py27]
+      for_envs: [py38, py37, py36, py35]
 
 - ${{ if startsWith(variables['Build.SourceBranch'], 'refs/tags/') }}:
   - template: publish-pypi.yml@tox

setup.cfg

@@ -1,12 +1,11 @@
-# -*- coding: utf-8 -*-
 [metadata]
 name = tox
 description = tox is a generic virtualenv management and test command line tool
 long_description = file: README.md
 long_description_content_type = text/markdown
 keywords = virtual, environments, isolated, testing
 maintainer = Bernat Gabor, Oliver Bestwalter, Anthony Asottile
-author = Holger Krekel, Oliver Bestwalter, Bernát Gábor and others
+author = Holger Krekel, Oliver Bestwalter, Bernat Gabor and others
 maintainer-email = tox-dev@python.org
 url = http://tox.readthedocs.org
 project_urls =

src/tox/config/cli/env_var.py

@@ -15,22 +15,24 @@ def get_env_var(key, of_type):
     :param of_type: the type we would like to convert it to
     :return:
     """
-    environ_key = "TOX_{}".format(key.upper())
-    if environ_key in os.environ:
-        value = os.environ[environ_key]
-        # noinspection PyBroadException
-        try:
-            source = "env var {}".format(environ_key)
-            of_type = CONVERT.to(raw=value, of_type=of_type)
-            return of_type, source
-        except Exception as exception:
-            logging.warning(
-                "env var %s=%r cannot be transformed to %r because %r",
-                environ_key,
-                value,
-                of_type,
-                exception,
-            )
+    key_upper = key.upper()
+    for fmt in ("TOX_{}", "TOX{}"):
+        environ_key = fmt.format(key_upper)
+        if environ_key in os.environ:
+            value = os.environ[environ_key]
+            # noinspection PyBroadException
+            try:
+                source = "env var {}".format(environ_key)
+                of_type = CONVERT.to(raw=value, of_type=of_type)
+                return of_type, source
+            except Exception as exception:
+                logging.warning(
+                    "env var %s=%r cannot be transformed to %r because %r",
+                    environ_key,
+                    value,
+                    of_type,
+                    exception,
+                )
 
 
 __all__ = ("get_env_var",)

src/tox/config/cli/parse.py

@@ -6,18 +6,28 @@
 
 
 def get_options(*args) -> Tuple[Parsed, List[str], Dict[str, Handler]]:
-    parsed, unknown = ToxParser.base().parse(args)
+    guess_verbosity = _get_base(args)
+    handlers, parsed, unknown = _get_core(args)
+    if guess_verbosity != parsed.verbosity:
+        setup_report(parsed.verbosity)  # pragma: no cover
+    return parsed, unknown, handlers
+
+
+def _get_base(args):
+    tox_parser = ToxParser.base()
+    parsed, unknown = tox_parser.parse(args)
     guess_verbosity = parsed.verbosity
     setup_report(guess_verbosity)
+    return guess_verbosity
+
 
+def _get_core(args):
     tox_parser = ToxParser.core()
     # noinspection PyUnresolvedReferences
     from tox.plugin.manager import MANAGER
 
     MANAGER.tox_add_option(tox_parser)
     tox_parser.fix_defaults()
     parsed, unknown = tox_parser.parse(args)
-    if guess_verbosity != parsed.verbosity:
-        setup_report(parsed.verbosity)  # pragma: no cover
     handlers = {k: p for k, (_, p) in tox_parser.handlers.items()}
-    return parsed, unknown, handlers
+    return handlers, parsed, unknown

src/tox/config/cli/parser.py

@@ -2,7 +2,7 @@
 import logging
 from argparse import SUPPRESS, Action, ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace
 from itertools import chain
-from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type, TypeVar, cast
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type, TypeVar
 
 from tox.plugin.util import NAME
 from tox.session.state import State
@@ -34,6 +34,7 @@ def fix_default(self, action: Action) -> None:
                 outcome = self.file_config.get(key, of_type=of_type)
             if outcome is not None:
                 action.default, action.default_source = outcome
+        # noinspection PyProtectedMember
         if isinstance(action, argparse._SubParsersAction):
             for values in action.choices.values():
                 values.fix_defaults()
@@ -42,6 +43,7 @@ def fix_default(self, action: Action) -> None:
     def get_type(action):
         of_type = getattr(action, "of_type", None)
         if of_type is None:
+            # noinspection PyProtectedMember
             if action.default is not None:
                 of_type = type(action.default)
             elif isinstance(action, argparse._StoreConstAction) and action.const is not None:
@@ -56,6 +58,7 @@ def __init__(self, prog: str) -> None:
         super(HelpFormatter, self).__init__(prog, max_help_position=42, width=240)
 
     def _get_help_string(self, action: Action) -> str:
+        # noinspection PyProtectedMember
         text = super()._get_help_string(action)
         if hasattr(action, "default_source"):
             default = " (default: %(default)s)"
@@ -85,11 +88,12 @@ def __init__(
         super().__init__(*args, **kwargs)
         if root is True:
             self._add_base_options()
-        self.handlers: Dict[str, Tuple[Any, Handler]] = {}
+        self.handlers = {}  # type:Dict[str, Tuple[Any, Handler]]
         if add_cmd is True:
             self._cmd = self.add_subparsers(
-                title="command", help="tox command to execute", dest="command", required=False
+                title="command", help="tox command to execute", dest="command"
             )
+            self._cmd.required = False
             self._cmd.default = "run"
 
         else:
@@ -140,22 +144,44 @@ def _add_base_options(self) -> None:
         self.fix_defaults()
 
     def parse(self, args: Sequence[str]) -> Tuple[Parsed, List[str]]:
-        arg_set = set(args)
-        if (
-            "-h" not in arg_set
-            and "--help" not in arg_set
-            and self._cmd is not None
-            and not (arg_set & set(self.handlers.keys()))
-        ):
-            global_args = set(
-                chain.from_iterable(
-                    i.option_strings for i in self._actions if hasattr(i, "option_strings")
-                )
+        args = self._inject_default_cmd(args)
+        result = Parsed()
+        _, unknown = super().parse_known_args(args, namespace=result)
+        return result, unknown
+
+    def _inject_default_cmd(self, args):
+        # we need to inject the command if not present and reorganize args left of the command
+        if self._cmd is None:  # no commands yet so must be all global, nothing to fix
+            return args
+        _global = {
+            k: v
+            for k, v in chain.from_iterable(
+                ((j, isinstance(i, argparse._StoreAction)) for j in i.option_strings)
+                for i in self._actions
+                if hasattr(i, "option_strings")
             )
-            first_differ = next((i for i, j in enumerate(args) if j not in global_args), 0)
-            new_args = list(args[:first_differ])
-            new_args.append(self._cmd.default or "run")
-            new_args.extend(args[first_differ:])
-            args = new_args
-        parsed, unknown = super().parse_known_args(args, namespace=Parsed())
-        return cast(Parsed, parsed), unknown
+        }
+        _global_single = {i[1:] for i in _global if len(i) == 2 and i.startswith("-")}
+        cmd_at = next((j for j, i in enumerate(args) if i in self._cmd.choices), None)
+        global_args, command_args = [], []
+        reorganize_to = cmd_at if cmd_at is not None else len(args)
+        at = 0
+        while at < reorganize_to:
+            arg = args[at]
+            needs_extra = False
+            is_global = False
+            if arg in _global:
+                needs_extra = _global[arg]
+                is_global = True
+            elif arg.startswith("-") and not (set(arg[1:]) - _global_single):
+                is_global = True
+            (global_args if is_global else command_args).append(arg)
+            at += 1
+            if needs_extra:
+                global_args.append(args[at])
+                at += 1
+        new_args = global_args
+        new_args.append(self._cmd.default if cmd_at is None else args[cmd_at])
+        new_args.extend(command_args)
+        new_args.extend(args[reorganize_to + 1 :])
+        return new_args

src/tox/config/sets.py

@@ -77,11 +77,16 @@ def __call__(self, src: Loader, conf: "Config"):
         return self._cache
 
     def __deepcopy__(self, memo):
-        result = type(self)(None, None, None, None, None)
-        orig_value = self._cache
-        result.__dict__ = deepcopy(self.__dict__)
-        if orig_value is _PLACE_HOLDER:
-            result._cache = _PLACE_HOLDER
+        # we should not copy the place holder as our checks would break
+        cls = self.__class__
+        result = cls.__new__(cls)
+        memo[id(self)] = result
+        for k, v in self.__dict__.items():
+            if k != "_cache" and v is _PLACE_HOLDER:
+                value = deepcopy(v, memo=memo)
+            else:
+                value = v
+            setattr(result, k, value)
         return result
 
     def __repr__(self):
@@ -98,7 +103,7 @@ def __repr__(self):
 class ConfigSet:
     def __init__(self, raw: Loader, conf: "Config"):
         self._raw = raw
-        self._defined: Dict[str, ConfigDefinition] = {}
+        self._defined = {}  # type:Dict[str, ConfigDefinition]
         self._conf = conf
         self._keys = OrderedDict()
         self._raw.setup_with_conf(self)

src/tox/config/source/api.py

@@ -8,7 +8,7 @@
 
 class Command:
     def __init__(self, args):
-        self.args: List[str] = args
+        self.args = args  # type:List[str]
 
     def __repr__(self):
         return "{}(args={!r})".format(type(self).__name__, self.args)
@@ -50,12 +50,12 @@ def to(self, raw, of_type):
     def _to_typing(self, raw, of_type):
         origin = getattr(of_type, "__origin__", None)
         if origin is not None:
-            result: Any = _NO_MAPPING
-            if origin == list:
+            result = _NO_MAPPING  # type: Any
+            if origin in (list, List):
                 result = [self.to(i, of_type.__args__[0]) for i in self.to_list(raw)]
-            elif origin == set:
+            elif origin in (set, Set):
                 result = {self.to(i, of_type.__args__[0]) for i in self.to_set(raw)}
-            elif origin == dict:
+            elif origin in (dict, Dict):
                 result = OrderedDict(
                     (self.to(k, of_type.__args__[0]), self.to(v, of_type.__args__[1]))
                     for k, v in self.to_dict(raw)
@@ -69,7 +69,7 @@ def _to_typing(self, raw, of_type):
                         result = self._to_typing(raw, new_type)
             if result is not _NO_MAPPING:
                 return result
-        raise TypeError("{} cannot cast to {}".format(raw, of_type))
+        raise TypeError("{} cannot cast to {!r}".format(raw, of_type))
 
     @staticmethod
     def to_str(value):
@@ -140,8 +140,8 @@ def found_keys(self) -> Set[str]:
 
 class Source(ABC):
     def __init__(self, core: Loader) -> None:
-        self.core: Loader = core
-        self._envs: Dict[str, Loader] = {}
+        self.core = core  # type: Loader
+        self._envs = {}  # type: Dict[str, Loader]
 
     @abstractmethod
     def envs(self, core_conf):

src/tox/config/source/ini/__init__.py

@@ -1,4 +1,5 @@
 from configparser import ConfigParser, SectionProxy
+from copy import deepcopy
 from itertools import chain
 from pathlib import Path
 from typing import Dict, List, Optional, Set
@@ -29,7 +30,19 @@ def __init__(self, path: Path) -> None:
             default_base=EnvList([]),
         )
         super().__init__(core)
-        self._envs: Dict[str, IniLoader] = {}
+        self._envs = {}  # type: Dict[str, IniLoader]
+
+    def __deepcopy__(self, memo):
+        # python < 3.7 cannot copy config parser
+        result = self.__class__.__new__(self.__class__)
+        memo[id(self)] = result
+        for k, v in self.__dict__.items():
+            if k != "_parser":
+                value = deepcopy(v, memo=memo)
+            else:
+                value = v
+            setattr(result, k, value)
+        return result
 
     def _get_section(self, key):
         if self._parser.has_section(key):
@@ -94,13 +107,26 @@ def __init__(
         self, section: Optional[SectionProxy], src: Ini, name: Optional[str], default_base: EnvList
     ) -> None:
         super().__init__(name)
-        self._section: Optional[SectionProxy] = section
-        self._src: Ini = src
-        self._default_base: EnvList = default_base
-        self._base: List[IniLoader] = []
+        self._section = section  # type:Optional[SectionProxy]
+        self._src = src  # type: Ini
+        self._default_base = default_base  # type:EnvList
+        self._base = []  # type:List[IniLoader]
+
+    def __deepcopy__(self, memo):
+        # python < 3.7 cannot copy config parser
+        result = self.__class__.__new__(self.__class__)
+        memo[id(self)] = result
+        for k, v in self.__dict__.items():
+            if k != "_section":
+                value = deepcopy(v, memo=memo)
+            else:
+                value = v
+            setattr(result, k, value)
+        return result
 
     def setup_with_conf(self, conf: ConfigSet):
-        def load_bases(values, conf):
+        # noinspection PyUnusedLocal
+        def load_bases(values, conf_):
             result = []
             for value in values:
                 name = value.lstrip(TEST_ENV_PREFIX)

src/tox/config/source/ini/factor.py

@@ -46,7 +46,7 @@ def explode_factor(group):
 
 def expand_factors(value):
     for line in value.split("\n"):
-        match = re.match(r"^((?P<factor_expr>[\w{}.!,-]+)\:\s+)?(?P<content>.*?)$", line)
+        match = re.match(r"^((?P<factor_expr>[\w{}.!,-]+):\s+)?(?P<content>.*?)$", line)
         groups = match.groupdict()
         factor_expr, content = groups["factor_expr"], groups["content"]
         if factor_expr is not None:
@@ -76,7 +76,7 @@ def expand_env_with_negation(value):
     for key, group in itertools.groupby(re.split(r"((?:{[^}]+\})+)|,", value), key=bool):
         if key:
             group_str = "".join(group).strip()
-            elements = re.split(r"\{([^}]+)\}", group_str)
+            elements = re.split(r"{([^}]+)\}", group_str)
             parts = [re.sub(r"\s+", "", elem).split(",") for elem in elements]
             for variant in itertools.product(*parts):
                 variant_str = "".join(variant)

src/tox/config/source/ini/replace.py

@@ -19,6 +19,7 @@
 
 
 def substitute_once(val, conf, name):
+    # noinspection PyTypeChecker
     return RE_ITEM_REF.sub(partial(_replace_match, conf, name), val)