Avoid modification of install_requires and extra_requires that deviates from core metadata

setuptools/command/_requirestxt.py

@@ -0,0 +1,127 @@
+"""Helper code used to generate ``requires.txt`` files in the egg-info directory.
+
+The ``requires.txt`` file has an specific format:
+    - Environment markers need to be part of the section headers and
+      should not be part of the requirement spec itself.
+
+See https://setuptools.pypa.io/en/latest/deprecated/python_eggs.html#requires-txt
+"""
+import io
+from collections import defaultdict
+from itertools import filterfalse
+from typing import Dict, List, Tuple, Mapping, TypeVar
+
+from .. import _reqs
+from ..extern.jaraco.text import yield_lines
+from ..extern.packaging.requirements import Requirement
+
+
+# dict can work as an ordered set
+_T = TypeVar("_T")
+_Ordered = Dict[_T, None]
+_ordered = dict
+_StrOrIter = _reqs._StrOrIter
+
+
+def _prepare(
+    install_requires: _StrOrIter, extras_require: Mapping[str, _StrOrIter]
+) -> Tuple[List[str], Dict[str, List[str]]]:
+    """Given values for ``install_requires`` and ``extras_require``
+    create modified versions in a way that can be written in ``requires.txt``
+    """
+    extras = _convert_extras_requirements(extras_require)
+    return _move_install_requirements_markers(install_requires, extras)
+
+
+def _convert_extras_requirements(
+    extras_require: _StrOrIter,
+) -> Mapping[str, _Ordered[Requirement]]:
+    """
+    Convert requirements in `extras_require` of the form
+    `"extra": ["barbazquux; {marker}"]` to
+    `"extra:{marker}": ["barbazquux"]`.
+    """
+    output: Mapping[str, _Ordered[Requirement]] = defaultdict(dict)
+    for section, v in extras_require.items():
+        # Do not strip empty sections.
+        output[section]
+        for r in _reqs.parse(v):
+            output[section + _suffix_for(r)].setdefault(r)
+
+    return output
+
+
+def _move_install_requirements_markers(
+    install_requires: _StrOrIter, extras_require: Mapping[str, _Ordered[Requirement]]
+) -> Tuple[List[str], Dict[str, List[str]]]:
+    """
+    The ``requires.txt`` file has an specific format:
+        - Environment markers need to be part of the section headers and
+          should not be part of the requirement spec itself.
+
+    Move requirements in ``install_requires`` that are using environment
+    markers ``extras_require``.
+    """
+
+    # divide the install_requires into two sets, simple ones still
+    # handled by install_requires and more complex ones handled by extras_require.
+
+    inst_reqs = list(_reqs.parse(install_requires))
+    simple_reqs = filter(_no_marker, inst_reqs)
+    complex_reqs = filterfalse(_no_marker, inst_reqs)
+    simple_install_requires = list(map(str, simple_reqs))
+
+    for r in complex_reqs:
+        extras_require[':' + str(r.marker)].setdefault(r)
+
+    expanded_extras = dict(
+        # list(dict.fromkeys(...))  ensures a list of unique strings
+        (k, list(dict.fromkeys(str(r) for r in map(_clean_req, v))))
+        for k, v in extras_require.items()
+    )
+
+    return simple_install_requires, expanded_extras
+
+
+def _suffix_for(req):
+    """Return the 'extras_require' suffix for a given requirement."""
+    return ':' + str(req.marker) if req.marker else ''
+
+
+def _clean_req(req):
+    """Given a Requirement, remove environment markers and return it"""
+    req.marker = None
+    return req
+
+
+def _no_marker(req):
+    return not req.marker
+
+
+def _write_requirements(stream, reqs):
+    lines = yield_lines(reqs or ())
+
+    def append_cr(line):
+        return line + '\n'
+
+    lines = map(append_cr, lines)
+    stream.writelines(lines)
+
+
+def write_requirements(cmd, basename, filename):
+    dist = cmd.distribution
+    data = io.StringIO()
+    install_requires, extras_require = _prepare(
+        dist.install_requires or (), dist.extras_require or {}
+    )
+    _write_requirements(data, install_requires)
+    for extra in sorted(extras_require):
+        data.write('\n[{extra}]\n'.format(**vars()))
+        _write_requirements(data, extras_require[extra])
+    cmd.write_or_delete_file("requirements", filename, data.getvalue())
+
+
+def write_setup_requirements(cmd, basename, filename):
+    data = io.StringIO()
+    _write_requirements(data, cmd.distribution.setup_requires)
+    cmd.write_or_delete_file("setup-requirements", filename, data.getvalue())

setuptools/command/egg_info.py

@@ -12,12 +12,12 @@
 import os
 import re
 import sys
-import io
 import time
 import collections
 
 from .._importlib import metadata
 from .. import _entry_points, _normalization
+from . import _requirestxt
 
 from setuptools import Command
 from setuptools.command.sdist import sdist
@@ -28,7 +28,6 @@
 from setuptools.glob import glob
 
 from setuptools.extern import packaging
-from setuptools.extern.jaraco.text import yield_lines
 from ..warnings import SetuptoolsDeprecationWarning
 
 
@@ -692,31 +691,9 @@ def warn_depends_obsolete(cmd, basename, filename):
     """
 
 
-def _write_requirements(stream, reqs):
-    lines = yield_lines(reqs or ())
-
-    def append_cr(line):
-        return line + '\n'
-
-    lines = map(append_cr, lines)
-    stream.writelines(lines)
-
-
-def write_requirements(cmd, basename, filename):
-    dist = cmd.distribution
-    data = io.StringIO()
-    _write_requirements(data, dist.install_requires)
-    extras_require = dist.extras_require or {}
-    for extra in sorted(extras_require):
-        data.write('\n[{extra}]\n'.format(**vars()))
-        _write_requirements(data, extras_require[extra])
-    cmd.write_or_delete_file("requirements", filename, data.getvalue())
-
-
-def write_setup_requirements(cmd, basename, filename):
-    data = io.StringIO()
-    _write_requirements(data, cmd.distribution.setup_requires)
-    cmd.write_or_delete_file("setup-requirements", filename, data.getvalue())
+# Export API used in entry_points
+write_requirements = _requirestxt.write_requirements
+write_setup_requirements = _requirestxt.write_setup_requirements
 
 
 def write_toplevel_names(cmd, basename, filename):

setuptools/config/_apply_pyprojecttoml.py

@@ -216,7 +216,7 @@ def _dependencies(dist: "Distribution", val: list, _root_dir):
 
 
 def _optional_dependencies(dist: "Distribution", val: dict, _root_dir):
-    existing = getattr(dist, "extras_require", {})
+    existing = getattr(dist, "extras_require", None) or {}
     _set_config(dist, "extras_require", {**existing, **val})
 
 
@@ -383,8 +383,8 @@ def _acessor(obj):
     "entry-points": _get_previous_entrypoints,
     "scripts": _get_previous_scripts,
     "gui-scripts": _get_previous_gui_scripts,
-    "dependencies": _some_attrgetter("_orig_install_requires", "install_requires"),
-    "optional-dependencies": _some_attrgetter("_orig_extras_require", "extras_require"),
+    "dependencies": _attrgetter("install_requires"),
+    "optional-dependencies": _attrgetter("extras_require"),
 }
 
 

setuptools/dist.py

@@ -8,7 +8,6 @@
 import re
 import sys
 import textwrap
-from collections import defaultdict
 from contextlib import suppress
 from email import message_from_file
 from glob import iglob
@@ -490,11 +489,6 @@ def __init__(self, attrs=None):
         # sdist (e.g. `version = file: VERSION.txt`)
         self._referenced_files: Set[str] = set()
 
-        # Save the original dependencies before they are processed into the egg format
-        self._orig_extras_require = {}
-        self._orig_install_requires = []
-        self._tmp_extras_require = defaultdict(OrderedSet)
-
         self.set_defaults = ConfigDiscovery(self)
 
         self._set_metadata_defaults(attrs)
@@ -575,81 +569,23 @@ def _finalize_requires(self):
         if getattr(self, 'python_requires', None):
             self.metadata.python_requires = self.python_requires
 
-        if getattr(self, 'extras_require', None):
-            # Save original before it is messed by _convert_extras_requirements
-            self._orig_extras_require = self._orig_extras_require or self.extras_require
+        self._normalize_requires()
+
+        if self.extras_require:
             for extra in self.extras_require.keys():
-                # Since this gets called multiple times at points where the
-                # keys have become 'converted' extras, ensure that we are only
-                # truly adding extras we haven't seen before here.
+                # Setuptools allows a weird "<name>:<env markers> syntax for extras
                 extra = extra.split(':')[0]
                 if extra:
                     self.metadata.provides_extras.add(extra)
 
-        if getattr(self, 'install_requires', None) and not self._orig_install_requires:
-            # Save original before it is messed by _move_install_requirements_markers
-            self._orig_install_requires = self.install_requires
-
-        self._convert_extras_requirements()
-        self._move_install_requirements_markers()
-
-    def _convert_extras_requirements(self):
-        """
-        Convert requirements in `extras_require` of the form
-        `"extra": ["barbazquux; {marker}"]` to
-        `"extra:{marker}": ["barbazquux"]`.
-        """
-        spec_ext_reqs = getattr(self, 'extras_require', None) or {}
-        tmp = defaultdict(OrderedSet)
-        self._tmp_extras_require = getattr(self, '_tmp_extras_require', tmp)
-        for section, v in spec_ext_reqs.items():
-            # Do not strip empty sections.
-            self._tmp_extras_require[section]
-            for r in _reqs.parse(v):
-                suffix = self._suffix_for(r)
-                self._tmp_extras_require[section + suffix].append(r)
-
-    @staticmethod
-    def _suffix_for(req):
-        """
-        For a requirement, return the 'extras_require' suffix for
-        that requirement.
-        """
-        return ':' + str(req.marker) if req.marker else ''
-
-    def _move_install_requirements_markers(self):
-        """
-        Move requirements in `install_requires` that are using environment
-        markers `extras_require`.
-        """
-
-        # divide the install_requires into two sets, simple ones still
-        # handled by install_requires and more complex ones handled
-        # by extras_require.
-
-        def is_simple_req(req):
-            return not req.marker
-
-        spec_inst_reqs = getattr(self, 'install_requires', None) or ()
-        inst_reqs = list(_reqs.parse(spec_inst_reqs))
-        simple_reqs = filter(is_simple_req, inst_reqs)
-        complex_reqs = itertools.filterfalse(is_simple_req, inst_reqs)
-        self.install_requires = list(map(str, simple_reqs))
-
-        for r in complex_reqs:
-            self._tmp_extras_require[':' + str(r.marker)].append(r)
-        self.extras_require = dict(
-            # list(dict.fromkeys(...))  ensures a list of unique strings
-            (k, list(dict.fromkeys(str(r) for r in map(self._clean_req, v))))
-            for k, v in self._tmp_extras_require.items()
-        )
-
-    def _clean_req(self, req):
-        """
-        Given a Requirement, remove environment markers and return it.
-        """
-        req.marker = None
-        return req
+    def _normalize_requires(self):
+        """Make sure requirement-related attributes exist and are normalized"""
+        install_requires = getattr(self, "install_requires", None) or []
+        extras_require = getattr(self, "extras_require", None) or {}
+        self.install_requires = list(map(str, _reqs.parse(install_requires)))
+        self.extras_require = {
+            k: list(map(str, _reqs.parse(v or []))) for k, v in extras_require.items()
+        }
 
     def _finalize_license_files(self):
         """Compute names of all license files which should be included."""

setuptools/tests/config/test_apply_pyprojecttoml.py

@@ -388,12 +388,12 @@ def test_optional_dependencies_dont_remove_env_markers(self, tmp_path):
         dist = makedist(tmp_path, install_requires=install_req)
         dist = pyprojecttoml.apply_configuration(dist, pyproject)
         assert "foo" in dist.extras_require
-        assert ':python_version < "3.7"' in dist.extras_require
         egg_info = dist.get_command_obj("egg_info")
         write_requirements(egg_info, tmp_path, tmp_path / "requires.txt")
         reqs = (tmp_path / "requires.txt").read_text(encoding="utf-8")
         assert "importlib-resources" in reqs
         assert "bar" in reqs
+        assert ':python_version < "3.7"' in reqs
 
     @pytest.mark.parametrize(
         "field,group", [("scripts", "console_scripts"), ("gui-scripts", "gui_scripts")]