Improve core metadata compliance for PKG-INFO

setuptools/_core_metadata.py

@@ -0,0 +1,258 @@
+"""
+Handling of Core Metadata for Python packages (including reading and writing).
+
+See: https://packaging.python.org/en/latest/specifications/core-metadata/
+"""
+import os
+import stat
+import textwrap
+from email import message_from_file
+from email.message import Message
+from tempfile import NamedTemporaryFile
+from typing import Optional, List
+
+from distutils.util import rfc822_escape
+
+from . import _normalization
+from .extern.packaging.markers import Marker
+from .extern.packaging.requirements import Requirement
+from .extern.packaging.version import Version
+from .warnings import SetuptoolsDeprecationWarning
+
+
+def get_metadata_version(self):
+    mv = getattr(self, 'metadata_version', None)
+    if mv is None:
+        mv = Version('2.1')
+        self.metadata_version = mv
+    return mv
+
+
+def rfc822_unescape(content: str) -> str:
+    """Reverse RFC-822 escaping by removing leading whitespaces from content."""
+    lines = content.splitlines()
+    if len(lines) == 1:
+        return lines[0].lstrip()
+    return '\n'.join((lines[0].lstrip(), textwrap.dedent('\n'.join(lines[1:]))))
+
+
+def _read_field_from_msg(msg: Message, field: str) -> Optional[str]:
+    """Read Message header field."""
+    value = msg[field]
+    if value == 'UNKNOWN':
+        return None
+    return value
+
+
+def _read_field_unescaped_from_msg(msg: Message, field: str) -> Optional[str]:
+    """Read Message header field and apply rfc822_unescape."""
+    value = _read_field_from_msg(msg, field)
+    if value is None:
+        return value
+    return rfc822_unescape(value)
+
+
+def _read_list_from_msg(msg: Message, field: str) -> Optional[List[str]]:
+    """Read Message header field and return all results as list."""
+    values = msg.get_all(field, None)
+    if values == []:
+        return None
+    return values
+
+
+def _read_payload_from_msg(msg: Message) -> Optional[str]:
+    value = msg.get_payload().strip()
+    if value == 'UNKNOWN' or not value:
+        return None
+    return value
+
+
+def read_pkg_file(self, file):
+    """Reads the metadata values from a file object."""
+    msg = message_from_file(file)
+
+    self.metadata_version = Version(msg['metadata-version'])
+    self.name = _read_field_from_msg(msg, 'name')
+    self.version = _read_field_from_msg(msg, 'version')
+    self.description = _read_field_from_msg(msg, 'summary')
+    # we are filling author only.
+    self.author = _read_field_from_msg(msg, 'author')
+    self.maintainer = None
+    self.author_email = _read_field_from_msg(msg, 'author-email')
+    self.maintainer_email = None
+    self.url = _read_field_from_msg(msg, 'home-page')
+    self.download_url = _read_field_from_msg(msg, 'download-url')
+    self.license = _read_field_unescaped_from_msg(msg, 'license')
+
+    self.long_description = _read_field_unescaped_from_msg(msg, 'description')
+    if self.long_description is None and self.metadata_version >= Version('2.1'):
+        self.long_description = _read_payload_from_msg(msg)
+    self.description = _read_field_from_msg(msg, 'summary')
+
+    if 'keywords' in msg:
+        self.keywords = _read_field_from_msg(msg, 'keywords').split(',')
+
+    self.platforms = _read_list_from_msg(msg, 'platform')
+    self.classifiers = _read_list_from_msg(msg, 'classifier')
+
+    # PEP 314 - these fields only exist in 1.1
+    if self.metadata_version == Version('1.1'):
+        self.requires = _read_list_from_msg(msg, 'requires')
+        self.provides = _read_list_from_msg(msg, 'provides')
+        self.obsoletes = _read_list_from_msg(msg, 'obsoletes')
+    else:
+        self.requires = None
+        self.provides = None
+        self.obsoletes = None
+
+    self.license_files = _read_list_from_msg(msg, 'license-file')
+
+
+def single_line(val):
+    """
+    Quick and dirty validation for Summary pypa/setuptools#1390.
+    """
+    if '\n' in val:
+        # TODO: Replace with `raise ValueError("newlines not allowed")`
+        # after reviewing #2893.
+        msg = "newlines are not allowed in `summary` and will break in the future"
+        SetuptoolsDeprecationWarning.emit("Invalid config.", msg)
+        # due_date is undefined. Controversial change, there was a lot of push back.
+        val = val.strip().split('\n')[0]
+    return val
+
+
+def write_pkg_info(self, base_dir):
+    """Write the PKG-INFO file into the release tree."""
+    temp = ""
+    final = os.path.join(base_dir, 'PKG-INFO')
+    try:
+        # Use a temporary file while writing to avoid race conditions
+        # (e.g. `importlib.metadata` reading `.egg-info/PKG-INFO`):
+        with NamedTemporaryFile("w", encoding="utf-8", dir=base_dir, delete=False) as f:
+            temp = f.name
+            self.write_pkg_file(f)
+        permissions = stat.S_IMODE(os.lstat(temp).st_mode)
+        os.chmod(temp, permissions | stat.S_IRGRP | stat.S_IROTH)
+        os.replace(temp, final)  # atomic operation.
+    finally:
+        if temp and os.path.exists(temp):
+            os.remove(temp)
+
+
+# Based on Python 3.5 version
+def write_pkg_file(self, file):  # noqa: C901  # is too complex (14)  # FIXME
+    """Write the PKG-INFO format data to a file object."""
+    version = self.get_metadata_version()
+
+    def write_field(key, value):
+        file.write("%s: %s\n" % (key, value))
+
+    write_field('Metadata-Version', str(version))
+    write_field('Name', self.get_name())
+    write_field('Version', self.get_version())
+
+    summary = self.get_description()
+    if summary:
+        write_field('Summary', single_line(summary))
+
+    optional_fields = (
+        ('Home-page', 'url'),
+        ('Download-URL', 'download_url'),
+        ('Author', 'author'),
+        ('Author-email', 'author_email'),
+        ('Maintainer', 'maintainer'),
+        ('Maintainer-email', 'maintainer_email'),
+    )
+
+    for field, attr in optional_fields:
+        attr_val = getattr(self, attr, None)
+        if attr_val is not None:
+            write_field(field, attr_val)
+
+    license = self.get_license()
+    if license:
+        write_field('License', rfc822_escape(license))
+
+    for project_url in self.project_urls.items():
+        write_field('Project-URL', '%s, %s' % project_url)
+
+    keywords = ','.join(self.get_keywords())
+    if keywords:
+        write_field('Keywords', keywords)
+
+    platforms = self.get_platforms() or []
+    for platform in platforms:
+        write_field('Platform', platform)
+
+    self._write_list(file, 'Classifier', self.get_classifiers())
+
+    # PEP 314
+    self._write_list(file, 'Requires', self.get_requires())
+    self._write_list(file, 'Provides', self.get_provides())
+    self._write_list(file, 'Obsoletes', self.get_obsoletes())
+
+    # Setuptools specific for PEP 345
+    if hasattr(self, 'python_requires'):
+        write_field('Requires-Python', self.python_requires)
+
+    # PEP 566
+    if self.long_description_content_type:
+        write_field('Description-Content-Type', self.long_description_content_type)
+
+    self._write_list(file, 'License-File', self.license_files or [])
+    _write_requirements(self, file)
+
+    long_description = self.get_long_description()
+    if long_description:
+        file.write("\n%s" % long_description)
+        if not long_description.endswith("\n"):
+            file.write("\n")
+
+
+def _write_requirements(self, file):
+    for req in self._normalized_install_requires:
+        file.write(f"Requires-Dist: {req}\n")
+
+    processed_extras = {}
+    for augmented_extra, reqs in self._normalized_extras_require.items():
+        # Historically, setuptools allows "augmented extras": `<extra>:<condition>`
+        unsafe_extra, _, condition = augmented_extra.partition(":")
+        unsafe_extra = unsafe_extra.strip()
+        extra = _normalization.safe_extra(unsafe_extra)
+
+        if extra:
+            _write_provides_extra(file, processed_extras, extra, unsafe_extra)
+        for req in reqs:
+            r = _include_extra(req, extra, condition.strip())
+            file.write(f"Requires-Dist: {r}\n")
+
+    return processed_extras
+
+
+def _include_extra(req: str, extra: str, condition: str) -> Requirement:
+    r = Requirement(req)
+    parts = (
+        f"({r.marker})" if r.marker else None,
+        f"({condition})" if condition else None,
+        f"extra == {extra!r}" if extra else None,
+    )
+    r.marker = Marker(" and ".join(x for x in parts if x))
+    return r
+
+
+def _write_provides_extra(file, processed_extras, safe, unsafe):
+    previous = processed_extras.get(safe)
+    if previous == unsafe:
+        SetuptoolsDeprecationWarning.emit(
+            'Ambiguity during "extra" normalization for dependencies.',
+            f"""
+            {previous!r} and {unsafe!r} normalize to the same value:\n
+                {safe!r}\n
+            In future versions, setuptools might halt the build process.
+            """,
+            see_url="https://peps.python.org/pep-0685/",
+        )
+    else:
+        processed_extras[safe] = unsafe
+        file.write(f"Provides-Extra: {safe}\n")

setuptools/_normalization.py

@@ -14,6 +14,7 @@
 # https://packaging.python.org/en/latest/specifications/core-metadata/#name
 _VALID_NAME = re.compile(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.I)
 _UNSAFE_NAME_CHARS = re.compile(r"[^A-Z0-9.]+", re.I)
+_NON_ALPHANUMERIC = re.compile(r"[^A-Z0-9]+", re.I)
 
 
 def safe_identifier(name: str) -> str:
@@ -92,6 +93,16 @@ def best_effort_version(version: str) -> str:
         return safe_name(v)
 
 
+def safe_extra(extra: str) -> str:
+    """Normalize extra name according to PEP 685
+    >>> safe_extra("_FrIeNdLy-._.-bArD")
+    'friendly-bard'
+    >>> safe_extra("FrIeNdLy-._.-bArD__._-")
+    'friendly-bard'
+    """
+    return _NON_ALPHANUMERIC.sub("-", extra).strip("-").lower()
+
+
 def filename_component(value: str) -> str:
     """Normalize each component of a filename (e.g. distribution/version part of wheel)
     Note: ``value`` needs to be already normalized.

setuptools/command/_requirestxt.py

@@ -11,7 +11,6 @@
 from itertools import filterfalse
 from typing import Dict, List, Tuple, Mapping, TypeVar
 
-from .. import _reqs
 from ..extern.jaraco.text import yield_lines
 from ..extern.packaging.requirements import Requirement
 
@@ -20,11 +19,11 @@
 _T = TypeVar("_T")
 _Ordered = Dict[_T, None]
 _ordered = dict
-_StrOrIter = _reqs._StrOrIter
 
 
 def _prepare(
-    install_requires: _StrOrIter, extras_require: Mapping[str, _StrOrIter]
+    install_requires: Dict[str, Requirement],
+    extras_require: Mapping[str, Dict[str, Requirement]],
 ) -> Tuple[List[str], Dict[str, List[str]]]:
     """Given values for ``install_requires`` and ``extras_require``
     create modified versions in a way that can be written in ``requires.txt``
@@ -34,7 +33,7 @@ def _prepare(
 
 
 def _convert_extras_requirements(
-    extras_require: _StrOrIter,
+    extras_require: Dict[str, Dict[str, Requirement]],
 ) -> Mapping[str, _Ordered[Requirement]]:
     """
     Convert requirements in `extras_require` of the form
@@ -45,14 +44,15 @@ def _convert_extras_requirements(
     for section, v in extras_require.items():
         # Do not strip empty sections.
         output[section]
-        for r in _reqs.parse(v):
+        for r in v.values():
             output[section + _suffix_for(r)].setdefault(r)
 
     return output
 
 
 def _move_install_requirements_markers(
-    install_requires: _StrOrIter, extras_require: Mapping[str, _Ordered[Requirement]]
+    install_requires: Dict[str, Requirement],
+    extras_require: Mapping[str, _Ordered[Requirement]],
 ) -> Tuple[List[str], Dict[str, List[str]]]:
     """
     The ``requires.txt`` file has an specific format:
@@ -66,7 +66,7 @@ def _move_install_requirements_markers(
     # divide the install_requires into two sets, simple ones still
     # handled by install_requires and more complex ones handled by extras_require.
 
-    inst_reqs = list(_reqs.parse(install_requires))
+    inst_reqs = install_requires.values()
     simple_reqs = filter(_no_marker, inst_reqs)
     complex_reqs = filterfalse(_no_marker, inst_reqs)
     simple_install_requires = list(map(str, simple_reqs))
@@ -90,8 +90,9 @@ def _suffix_for(req):
 
 def _clean_req(req):
     """Given a Requirement, remove environment markers and return it"""
-    req.marker = None
-    return req
+    r = Requirement(str(req))  # create a copy before modifying.
+    r.marker = None
+    return r
 
 
 def _no_marker(req):
@@ -110,9 +111,10 @@ def append_cr(line):
 
 def write_requirements(cmd, basename, filename):
     dist = cmd.distribution
+    meta = dist.metadata
     data = io.StringIO()
     install_requires, extras_require = _prepare(
-        dist.install_requires or (), dist.extras_require or {}
+        meta._normalized_install_requires, meta._normalized_extras_require
     )
     _write_requirements(data, install_requires)
     for extra in sorted(extras_require):