reduce cost of large variant matrix (#5392)

* discard unused variants before copying metadata

when variant matrix is large and mostly unused (as in conda-forge),
the length of input_variants may be several thousand
when only a few are actually used.

This causes `get_loop_vars` and `metadata.copy()` to become very expensive.

* try reducing with all used vars instead of loop vars

should reduce less

* perf: copy distributed variants list after subsetting

vastly reduces the number of copies computed for large variant matrices

* perf: pass used_vars subset to get_loop_vars

rather than computing all loop vars and then intersecting,
only consider relevant keys when computing loop vars

reduces get_used_loop_vars from O(n_vars * n_variants) to O(n_used_vars * n_variants)

* remove redundant deepcopy of config.variant

config.copy already copies this, no need to do it twice in metadata.copy

* add config.copy_variants method

to avoid calling pickle in too many places

* Update news/5392-variant-copy

* Add benchmark test for `render_recipe` (#5490)

---------

Co-authored-by: Matthew R. Becker <beckermr@users.noreply.github.com>
Co-authored-by: Bianca Henderson <beeankha@gmail.com>
Co-authored-by: Ken Odegard <kodegard@anaconda.com>

conda_build/config.py

@@ -31,7 +31,9 @@
 
 if TYPE_CHECKING:
     from pathlib import Path
-    from typing import Any
+    from typing import Any, TypeVar
+
+    T = TypeVar("T")
 
 invocation_time = ""
 
@@ -821,14 +823,23 @@ def clean_pkgs(self):
 
     def copy(self) -> Config:
         new = copy.copy(self)
-        # Use picke.loads(pickle.dumps(...) as a faster copy.deepcopy alternative.
-        new.variant = pickle.loads(pickle.dumps(self.variant, pickle.HIGHEST_PROTOCOL))
+        new.variant = self._copy_variants(self.variant)
         if hasattr(self, "variants"):
-            new.variants = pickle.loads(
-                pickle.dumps(self.variants, pickle.HIGHEST_PROTOCOL)
-            )
+            new.variants = self.copy_variants()
         return new
 
+    def _copy_variants(self, variant_or_list: T) -> T:
+        """Efficient deep copy used for variant dicts and lists"""
+        # Use pickle.loads(pickle.dumps(...) as a faster copy.deepcopy alternative.
+        return pickle.loads(pickle.dumps(variant_or_list, pickle.HIGHEST_PROTOCOL))
+
+    def copy_variants(self) -> list[dict] | None:
+        """Return deep copy of the variants list, if any"""
+        if getattr(self, "variants", None) is not None:
+            return self._copy_variants(self.variants)
+        else:
+            return None
+
     # context management - automatic cleanup if self.dirty or self.keep_old_work is not True
     def __enter__(self):
         pass

conda_build/metadata.py

@@ -2288,7 +2288,6 @@ def validate_features(self):
     def copy(self: Self) -> MetaData:
         new = copy.copy(self)
         new.config = self.config.copy()
-        new.config.variant = copy.deepcopy(self.config.variant)
         new.meta = copy.deepcopy(self.meta)
         new.type = getattr(
             self, "type", "conda_v2" if self.config.conda_pkg_format == "2" else "conda"
@@ -2672,15 +2671,16 @@ def get_output_metadata_set(
         _check_run_constrained(output_tuples)
         return output_tuples
 
-    def get_loop_vars(self):
-        return get_vars(getattr(self.config, "input_variants", self.config.variants))
+    def get_loop_vars(self, subset=None):
+        return get_vars(
+            getattr(self.config, "input_variants", self.config.variants), subset=subset
+        )
 
     def get_used_loop_vars(self, force_top_level=False, force_global=False):
-        loop_vars = self.get_loop_vars()
         used_vars = self.get_used_vars(
             force_top_level=force_top_level, force_global=force_global
         )
-        return set(loop_vars).intersection(used_vars)
+        return self.get_loop_vars(subset=used_vars)
 
     def get_rendered_recipe_text(
         self, permit_undefined_jinja=False, extract_pattern=None

conda_build/render.py

@@ -835,12 +835,20 @@ def distribute_variants(
     used_variables = metadata.get_used_loop_vars(force_global=False)
     top_loop = metadata.get_reduced_variant_set(used_variables)
 
+    # defer potentially expensive copy of input variants list
+    # until after reduction of the list for each variant
+    # since the initial list can be very long
+    all_variants = metadata.config.variants
+    metadata.config.variants = []
+
     for variant in top_loop:
         from .build import get_all_replacements
 
         get_all_replacements(variant)
         mv = metadata.copy()
         mv.config.variant = variant
+        # start with shared list:
+        mv.config.variants = all_variants
 
         pin_run_as_build = variant.get("pin_run_as_build", {})
         if mv.numpy_xx and "numpy" not in pin_run_as_build:
@@ -860,6 +868,9 @@ def distribute_variants(
                 )
                 or mv.config.variants
             )
+        # copy variants before we start modifying them,
+        # but after we've reduced the list via the conform_dict filter
+        mv.config.variants = mv.config.copy_variants()
         get_all_replacements(mv.config.variants)
         pin_run_as_build = variant.get("pin_run_as_build", {})
         if mv.numpy_xx and "numpy" not in pin_run_as_build:

conda_build/variants.py

@@ -700,7 +700,10 @@ def get_package_variants(recipedir_or_metadata, config=None, variants=None):
     return filter_combined_spec_to_used_keys(combined_spec, specs=specs)
 
 
-def get_vars(variants: Iterable[dict[str, Any]]) -> set[str]:
+def get_vars(
+    variants: Iterable[dict[str, Any]],
+    subset: set[str] | None = None,
+) -> set[str]:
     """For purposes of naming/identifying, provide a way of identifying which variables contribute
     to the matrix dimensionality"""
     first, *others = variants
@@ -710,10 +713,12 @@ def get_vars(variants: Iterable[dict[str, Any]]) -> set[str]:
         "ignore_version",
         *ensure_list(first.get("extend_keys")),
     }
+    to_consider = set(first)
+    if subset is not None:
+        to_consider.intersection_update(subset)
+    to_consider.difference_update(special_keys)
     return {
-        var
-        for var in set(first) - special_keys
-        if any(first[var] != other[var] for other in others)
+        var for var in to_consider if any(first[var] != other[var] for other in others)
     }
 
 

news/5392-variant-copy

@@ -0,0 +1,3 @@
+### Enhancements
+
+* Reduce render time when there is a large number of unused variants. (#5392)