Piggyback migrator for stdlib-migration

conda_forge_tick/auto_tick.py

@@ -84,6 +84,7 @@
     PipWheelMigrator,
     QtQtMainMigrator,
     Replacement,
+    StdlibMigrator,
     UpdateCMakeArgsMigrator,
     UpdateConfigSubGuessMigrator,
     Version,
@@ -684,6 +685,10 @@
         piggy_back_migrations.append(JpegTurboMigrator())
     if migration_name == "boost_cpp_to_libboost":
         piggy_back_migrations.append(LibboostMigrator())
+    if migration_name == "boost1840":
+        # testing phase: only a single migration
+        # TODO: piggyback for all migrations
+        piggy_back_migrations.append(StdlibMigrator())
     cycles = list(nx.simple_cycles(total_graph))
     migrator = MigrationYaml(
         migration_yaml,

conda_forge_tick/migrators/__init__.py

@@ -14,6 +14,7 @@
     UpdateCMakeArgsMigrator,
     UpdateConfigSubGuessMigrator,
 )
+from .cstdlib import StdlibMigrator
 from .dep_updates import DependencyUpdateMigrator
 from .duplicate_lines import DuplicateLinesCleanup
 from .extra_jinj2a_keys_cleanup import ExtraJinja2KeysCleanup

conda_forge_tick/migrators/cstdlib.py

@@ -0,0 +1,121 @@
+import os
+import re
+
+from conda_forge_tick.migrators.core import MiniMigrator
+from conda_forge_tick.migrators.libboost import _slice_into_output_sections
+
+pat_stub = re.compile(r"(c|cxx|fortran)_compiler_stub")
+pat_compiler = re.compile(
+    r"(?P<indent>\s*)-\s*"
+    r"(?P<compiler>\{\{\s*compiler\([\"\'](c|cxx|fortran)[\"\']\)\s*\}\})"
+    r"\s*(?P<selector>\#\s+\[[\w\s]+\])?"
+)
+pat_stdlib = re.compile(r".*\{\{\s*stdlib\([\"\']c[\"\']\)\s*\}\}.*")
+
+
+def _process_section(name, attrs, lines):
+    """
+    Migrate requirements per section.
+
+    We want to migrate as follows:
+    - if there's _any_ `{{ stdlib("c") }}` in the recipe, abort (consider it migrated)
+    - if there's `{{ compiler("c") }}` in build, add `{{ stdlib("c") }}` in host
+    - where there's no host-section, add it
+    """
+    outputs = attrs["meta_yaml"].get("outputs", [])
+    global_reqs = attrs["meta_yaml"].get("requirements", {})
+    if name == "global":
+        reqs = global_reqs
+    else:
+        filtered = [o for o in outputs if o["name"] == name]
+        if len(filtered) == 0:
+            raise RuntimeError(f"Could not find output {name}!")
+        reqs = filtered[0].get("requirements", {})
+
+    build_reqs = reqs.get("build", set()) or set()
+    global_build_reqs = global_reqs.get("build", set()) or set()
+
+    # either there's a compiler in the output we're processing, or the
+    # current output has no build-section but relies on the global one
+    needs_stdlib = any(pat_stub.search(x or "") for x in build_reqs)
+    needs_stdlib |= not bool(build_reqs) and any(
+        pat_stub.search(x or "") for x in global_build_reqs
+    )
+
+    if not needs_stdlib:
+        # no change
+        return lines
+
+    line_build = line_compiler = line_host = line_run = line_constrain = line_test = 0
+    indent = selector = ""
+    for i, line in enumerate(lines):
+        if re.match(r".*build:.*", line):
+            # always update this, as requirements.build follows build.{number,...}
+            line_build = i
+        elif pat_compiler.search(line):
+            line_compiler = i
+            indent = pat_compiler.match(line).group("indent")
+            selector = pat_compiler.match(line).group("selector") or ""
+        elif re.match(r".*host:.*", line):
+            line_host = i
+        elif re.match(r".*run:.*", line):
+            line_run = i
+        elif re.match(r".*run_constrained:.*", line):
+            line_constrain = i
+        elif re.match(r".*test:.*", line):
+            line_test = i
+            # ensure we don't read past test section (may contain unrelated deps)
+            break
+
+    if indent == "":
+        # no compiler in current output; take first line of section as reference (without last \n);
+        # ensure it works for both global build section as well as for `- name: <output>`.
+        indent = (
+            re.sub(r"^([\s\-]*).*", r"\1", lines[0][:-1]).replace("-", " ") + " " * 4
+        )
+
+    to_insert = indent + '- {{ stdlib("c") }}' + selector + "\n"
+    if line_host == 0:
+        # no host section, need to add it
+        to_insert = indent[:-2] + "host:\n" + to_insert
+
+    if line_host == 0 and line_run == 0:
+        # neither host nor run section; insert before
+        # run_constrained (if it exists) else test section
+        line_insert = line_constrain or line_test
+        if not line_insert:
+            raise RuntimeError("Don't know where to insert host section!")
+    elif line_host == 0:
+        # no host section, insert before run
+        line_insert = line_run
+    else:
+        # by default, we insert as first host dependency
+        line_insert = line_host + 1
+
+    return lines[:line_insert] + [to_insert] + lines[line_insert:]
+
+
+class StdlibMigrator(MiniMigrator):
+    def filter(self, attrs, not_bad_str_start=""):
+        lines = attrs["raw_meta_yaml"].splitlines()
+        already_migrated = any(pat_stdlib.search(line) for line in lines)
+        has_compiler = any(pat_compiler.search(line) for line in lines)
+        # filter() returns True if we _don't_ want to migrate
+        return already_migrated or not has_compiler
+
+    def migrate(self, recipe_dir, attrs, **kwargs):
+        outputs = attrs["meta_yaml"].get("outputs", [])
+
+        fname = os.path.join(recipe_dir, "meta.yaml")
+        if os.path.exists(fname):
+            with open(fname) as fp:
+                lines = fp.readlines()
+
+            new_lines = []
+            sections = _slice_into_output_sections(lines, attrs)
+            for name, section in sections.items():
+                # _process_section returns list of lines already
+                new_lines += _process_section(name, attrs, section)
+
+            with open(fname, "w") as fp:
+                fp.write("".join(new_lines))

conda_forge_tick/utils.py

@@ -34,6 +34,7 @@
 
 PACKAGE_STUBS = [
     "_compiler_stub",
+    "_stdlib_stub",
     "subpackage_stub",
     "compatible_pin_stub",
     "cdt_stub",
@@ -43,6 +44,7 @@
     os=os,
     environ=defaultdict(str),
     compiler=lambda x: x + "_compiler_stub",
+    stdlib=lambda x: x + "_stdlib_stub",
     pin_subpackage=lambda *args, **kwargs: args[0],
     pin_compatible=lambda *args, **kwargs: args[0],
     cdt=lambda *args, **kwargs: "cdt_stub",
@@ -61,6 +63,7 @@ def _munge_dict_repr(d):
     os=os,
     environ=defaultdict(str),
     compiler=lambda x: x + "_compiler_stub",
+    stdlib=lambda x: x + "_stdlib_stub",
     # The `max_pin, ` stub is so we know when people used the functions
     # to create the pins
     pin_subpackage=lambda *args, **kwargs: _munge_dict_repr(

tests/test_stdlib.py

@@ -0,0 +1,50 @@
+import os
+
+import pytest
+from flaky import flaky
+from test_migrators import run_test_migration
+
+from conda_forge_tick.migrators import StdlibMigrator, Version
+
+TEST_YAML_PATH = os.path.join(os.path.dirname(__file__), "test_yaml")
+
+
+STDLIB = StdlibMigrator()
+VERSION_WITH_STDLIB = Version(
+    set(),
+    piggy_back_migrations=[STDLIB],
+)
+
+
+@pytest.mark.parametrize(
+    "feedstock,new_ver",
+    [
+        ("arrow", "1.10.0"),
+        ("polars", "1.10.0"),
+        # test that we skip recipes that already contain a {{ stdlib("c") }}
+        ("skip_migration", "1.10.0"),
+    ],
+)
+def test_stdlib(feedstock, new_ver, tmpdir):
+    before = f"stdlib_{feedstock}_before_meta.yaml"
+    with open(os.path.join(TEST_YAML_PATH, before)) as fp:
+        in_yaml = fp.read()
+
+    after = f"stdlib_{feedstock}_after_meta.yaml"
+    with open(os.path.join(TEST_YAML_PATH, after)) as fp:
+        out_yaml = fp.read()
+
+    run_test_migration(
+        m=VERSION_WITH_STDLIB,
+        inp=in_yaml,
+        output=out_yaml,
+        kwargs={"new_version": new_ver},
+        prb="Dependencies have been updated if changed",
+        mr_out={
+            "migrator_name": "Version",
+            "migrator_version": VERSION_WITH_STDLIB.migrator_version,
+            "version": new_ver,
+        },
+        tmpdir=tmpdir,
+        should_filter=False,
+    )