feat: Support multiple extras in requirements constraint advertisement (

#379)

Update the base template's setup.py to support the full syntax of package
extras.

- Support commas and whitespace inside extras declarations, e.g.
  `some_package[extra_one, extra_two]`
- Raise exception if we name packages differently in requirement and
  constraint files, including presence/absence of extras.

CHANGELOG.rst

@@ -5,6 +5,15 @@ Change Log
    This file loosely adheres to the structure of https://keepachangelog.com/,
    but in reStructuredText instead of Markdown.
 
+2023-08-16
+**********
+
+Changed
+=======
+
+- In setup.py, support advertising constraints on packages with multiple extras
+- Fail packaging if requirements are named differently in different places or have different extras listed
+
 2023-08-11
 **********
 

python-template/{{cookiecutter.placeholder_repo_name}}/setup.py

@@ -36,17 +36,45 @@ def load_requirements(*requirements_paths):
     with -c in the requirements files.
     Returns a list of requirement strings.
     """
+    # e.g. {"django": "Django", "confluent-kafka": "confluent_kafka[avro]"}
+    by_canonical_name = {}
+
+    def check_name_consistent(package):
+        """
+        Raise exception if package is named different ways.
+
+        This ensures that packages are named consistently so we can match
+        constraints to packages. It also ensures that if we require a package
+        with extras we don't constrain it without mentioning the extras (since
+        that too would interfere with matching constraints.)
+        """
+        canonical = package.lower().replace('_', '-').split('[')[0]
+        seen_spelling = by_canonical_name.get(canonical)
+        if seen_spelling is None:
+            by_canonical_name[canonical] = package
+        elif seen_spelling != package:
+            raise Exception(
+                f'Encountered both "{seen_spelling}" and "{package}" in requirements '
+                'and constraints files; please use just one or the other.'
+            )
+
     requirements = {}
     constraint_files = set()
 
     # groups "pkg<=x.y.z,..." into ("pkg", "<=x.y.z,...")
-    requirement_line_regex = re.compile(r"([a-zA-Z0-9-_.\[\]]+)([<>=][^#\s]+)?")
+    re_package_name_base_chars = r"a-zA-Z0-9\-_."  # chars allowed in base package name
+    # Two groups: name[maybe,extras], and optionally a constraint
+    requirement_line_regex = re.compile(
+        r"([%s]+(?:\[[%s,\s]+\])?)([<>=][^#\s]+)?"
+        % (re_package_name_base_chars, re_package_name_base_chars)
+    )
 
     def add_version_constraint_or_raise(current_line, current_requirements, add_if_not_present):
         regex_match = requirement_line_regex.match(current_line)
         if regex_match:
             package = regex_match.group(1)
             version_constraints = regex_match.group(2)
+            check_name_consistent(package)
             existing_version_constraints = current_requirements.get(package, None)
             # It's fine to add constraints to an unconstrained package,
             # but raise an error if there are already constraints in place.

scripts/update_setup_py.sh

@@ -50,8 +50,8 @@ fi
 echo -e "[ARCHBOM-1772](https://openedx.atlassian.net/browse/ARCHBOM-1772)
  Update setup.py to use constraint files when generating requirements files for packaging and distribution.
  PR generated automatically with Jenkins job cleanup-python-code. " > .git/cleanup-python-code-description
-echo -e "\nResult of running \`python setup.py bdist_wheel\` before applying fix (in .egg-info/requires.txt)\: \n" >> .git/cleanup-python-code-description
+echo -e "\nResult of running \`python setup.py bdist_wheel\` before applying fix (in .egg-info/requires.txt): \n" >> .git/cleanup-python-code-description
 cat ./update-setup-tmp/old_requires.txt >> .git/cleanup-python-code-description
-echo -e "\nResult of running \`python setup.py bdist_wheel\` after applying fix (in .egg-info/requires.txt)\: \n" >> .git/cleanup-python-code-description
+echo -e "\nResult of running \`python setup.py bdist_wheel\` after applying fix (in .egg-info/requires.txt): \n" >> .git/cleanup-python-code-description
 cat "$(pwd)/$wheel_dir/requires.txt" >> .git/cleanup-python-code-description
 rm -rf update-setup-tmp

scripts/update_setup_py_load_requirements.yaml

@@ -18,17 +18,45 @@ rules:
         """
         # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why.
 
+        # e.g. {"django": "Django", "confluent-kafka": "confluent_kafka[avro]"}
+        by_canonical_name = {}
+
+        def check_name_consistent(package):
+            """
+            Raise exception if package is named different ways.
+
+            This ensures that packages are named consistently so we can match
+            constraints to packages. It also ensures that if we require a package
+            with extras we don't constrain it without mentioning the extras (since
+            that too would interfere with matching constraints.)
+            """
+            canonical = package.lower().replace('_', '-').split('[')[0]
+            seen_spelling = by_canonical_name.get(canonical)
+            if seen_spelling is None:
+                by_canonical_name[canonical] = package
+            elif seen_spelling != package:
+                raise Exception(
+                    f'Encountered both "{seen_spelling}" and "{package}" in requirements '
+                    'and constraints files; please use just one or the other.'
+                )
+
         requirements = {}
         constraint_files = set()
 
         # groups "pkg<=x.y.z,..." into ("pkg", "<=x.y.z,...")
-        requirement_line_regex = re.compile(r"([a-zA-Z0-9-_.\[\]]+)([<>=][^#\s]+)?")
+        re_package_name_base_chars = r"a-zA-Z0-9\-_."  # chars allowed in base package name
+        # Two groups: name[maybe,extras], and optionally a constraint
+        requirement_line_regex = re.compile(
+            r"([%s]+(?:\[[%s,\s]+\])?)([<>=][^#\s]+)?"
+            % (re_package_name_base_chars, re_package_name_base_chars)
+        )
 
         def add_version_constraint_or_raise(current_line, current_requirements, add_if_not_present):
             regex_match = requirement_line_regex.match(current_line)
             if regex_match:
                 package = regex_match.group(1)
                 version_constraints = regex_match.group(2)
+                check_name_consistent(package)
                 existing_version_constraints = current_requirements.get(package, None)
                 # It's fine to add constraints to an unconstrained package,
                 # but raise an error if there are already constraints in place.