Ephemeral packages

src/rez/cli/context.py

@@ -57,6 +57,10 @@ def setup_parser(parser, completions=False):
     parser.add_argument(
         "-g", "--graph", action="store_true",
         help="display the resolve graph as an image")
+    parser.add_argument(
+        "-d", "--dependency-graph", action="store_true",
+        help="display the (simpler) dependency graph. Works in combination "
+        "with other graph options")
     parser.add_argument(
         "--pg", "--print-graph", dest="print_graph", action="store_true",
         help="print the resolve graph as a string")
@@ -120,7 +124,10 @@ def command(opts, parser, extra_arg_groups=None):
 
     def _graph():
         if rc.has_graph:
-            return rc.graph(as_dot=True)
+            if opts.dependency_graph:
+                return rc.get_dependency_graph(as_dot=True)
+            else:
+                return rc.graph(as_dot=True)
         else:
             print("The context does not contain a graph.", file=sys.stderr)
             sys.exit(1)
@@ -155,12 +162,12 @@ def _graph():
         elif opts.print_graph:
             gstr = _graph()
             print(gstr)
-        elif opts.graph or opts.write_graph:
+        elif opts.graph or opts.dependency_graph or opts.write_graph:
             gstr = _graph()
             if opts.prune_pkg:
                 req = PackageRequest(opts.prune_pkg)
                 gstr = prune_graph(gstr, req.name)
-            func = view_graph if opts.graph else save_graph
+            func = view_graph if (opts.graph or opts.dependency_graph) else save_graph
             func(gstr, dest_file=opts.write_graph)
         else:
             rc.print_info(verbosity=opts.verbose,

src/rez/config.py

@@ -347,6 +347,7 @@ def _parse_env_var(self, value):
     "heading_styles":                               OptionalStrList,
     "local_styles":                                 OptionalStrList,
     "implicit_styles":                              OptionalStrList,
+    "ephemeral_styles":                             OptionalStrList,
     "alias_styles":                                 OptionalStrList,
     "memcached_uri":                                OptionalStrList,
     "pip_extra_args":                               OptionalStrList,
@@ -386,6 +387,8 @@ def _parse_env_var(self, value):
     "local_back":                                   OptionalStr,
     "implicit_fore":                                OptionalStr,
     "implicit_back":                                OptionalStr,
+    "ephemeral_fore":                               OptionalStr,
+    "ephemeral_back":                               OptionalStr,
     "alias_fore":                                   OptionalStr,
     "alias_back":                                   OptionalStr,
     "package_preprocess_function":                  OptionalStrOrFunction,

src/rez/resolved_context.py

@@ -6,9 +6,10 @@
 from rez.resolver import Resolver, ResolverStatus
 from rez.system import system
 from rez.config import config
-from rez.util import shlex_join, dedup, is_non_string_iterable
+from rez.util import dedup, is_non_string_iterable
 from rez.utils.sourcecode import SourceCodeError
-from rez.utils.colorize import critical, heading, local, implicit, Printer
+from rez.utils.colorize import critical, heading, local, implicit, Printer, \
+    ephemeral as ephemeral_color
 from rez.utils.formatting import columnise, PackageRequest, ENV_VAR_REGEX, \
     header_comment, minor_header_comment
 from rez.utils.data_utils import deep_del
@@ -18,7 +19,7 @@
 from rez.backport.shutilwhich import which
 from rez.rex import RexExecutor, Python, OutputStyle
 from rez.rex_bindings import VersionBinding, VariantBinding, \
-    VariantsBinding, RequirementsBinding
+    VariantsBinding, RequirementsBinding, EphemeralsBinding, intersects
 from rez import package_order
 from rez.packages import get_variant, iter_packages
 from rez.package_filter import PackageFilterList
@@ -30,6 +31,7 @@
 from rez.utils.graph_utils import write_dot, write_compacted, read_graph_from_string
 from rez.vendor.six import six
 from rez.vendor.version.version import VersionRange
+from rez.vendor.version.requirement import Requirement
 from rez.vendor.enum import Enum
 from rez.vendor import yaml
 from rez.utils import json
@@ -121,7 +123,7 @@ class ResolvedContext(object):
     command within a configured python namespace, without spawning a child
     shell.
     """
-    serialize_version = (4, 5)
+    serialize_version = (4, 6)
     tmpdir_manager = TempDirs(config.context_tmpdir, prefix="rez_context_")
 
     context_tracking_payload = None
@@ -255,6 +257,7 @@ def __init__(self, package_requests, verbosity=0, timestamp=None,
         # resolve results
         self.status_ = ResolverStatus.pending
         self._resolved_packages = None
+        self._resolved_ephemerals = None
         self.failure_description = None
         self.graph_string = None
         self.graph_ = None
@@ -313,11 +316,12 @@ def _package_load_callback(package):
 
         if self.status_ == ResolverStatus.solved:
             self._resolved_packages = []
-
             for variant in resolver.resolved_packages:
                 variant.set_context(self)
                 self._resolved_packages.append(variant)
 
+            self._resolved_ephemerals = resolver.resolved_ephemerals
+
         # track context usage
         if config.context_tracking_host:
             data = self.to_dict(fields=config.context_tracking_context_fields)
@@ -374,6 +378,15 @@ def resolved_packages(self):
         """
         return self._resolved_packages
 
+    @property
+    def resolved_ephemerals(self):
+        """Get non-conflict ephemerals in the resolve.
+
+        Returns:
+            List of `Requirement` objects, or None if the resolve failed.
+        """
+        return self._resolved_ephemerals
+
     def set_load_path(self, path):
         """Set the path that this context was reportedly loaded from.
 
@@ -386,13 +399,15 @@ def set_load_path(self, path):
     def __eq__(self, other):
         """Equality test.
 
-        Two contexts are considered equal if they have a equivalent request,
+        Two contexts are considered equal if they have an equivalent request,
         and an equivalent resolve. Other details, such as timestamp, are not
         considered.
         """
-        return (isinstance(other, ResolvedContext)
-                and other.requested_packages(True) == self.requested_packages(True)
-                and other.resolved_packages == self.resolved_packages)
+        return (
+            isinstance(other, ResolvedContext) and
+            other.requested_packages(True) == self.requested_packages(True) and
+            other.resolved_packages == self.resolved_packages
+        )
 
     def __hash__(self):
         list_ = []
@@ -767,8 +782,12 @@ def _rt(t):
         rows = []
         colors = []
         for request in self._package_requests:
-            rows.append((str(request), ""))
-            colors.append(None)
+            if request.name.startswith('.'):
+                rows.append((str(request), "(ephemeral)"))
+                colors.append(ephemeral_color)
+            else:
+                rows.append((str(request), ""))
+                colors.append(None)
 
         for request in self.implicit_packages:
             rows.append((str(request), "(implicit)"))
@@ -823,6 +842,13 @@ def _rt(t):
             rows.append((pkg.qualified_package_name, location, t))
             colors.append(col)
 
+        # add ephemerals to end of resolved packages list
+        ephemerals = self.resolved_ephemerals or []
+        ephemerals = sorted(ephemerals, key=lambda x: x.name)
+        for req in ephemerals:
+            rows.append((str(req), '', "(ephemeral)"))
+            colors.append(ephemeral_color)
+
         for col, line in zip(colors, columnise(rows)):
             _pr(line, col)
 
@@ -936,7 +962,7 @@ def _check(self, *nargs, **kwargs):
         return _check
 
     @_on_success
-    def get_dependency_graph(self):
+    def get_dependency_graph(self, as_dot=False):
         """Generate the dependency graph.
 
         The dependency graph is a simpler subset of the resolve graph. It
@@ -949,7 +975,14 @@ def get_dependency_graph(self):
         """
         from rez.vendor.pygraph.classes.digraph import digraph
 
+        # add nodes
         nodes = {}
+        for variant in self._resolved_packages:
+            nodes[variant.name] = variant.qualified_package_name
+        for ephemeral in self._resolved_ephemerals:
+            nodes[ephemeral.name] = str(ephemeral)
+
+        # add edges
         edges = set()
         for variant in self._resolved_packages:
             nodes[variant.name] = variant.qualified_package_name
@@ -968,7 +1001,11 @@ def get_dependency_graph(self):
             g.add_node(name, attrs=attrs + [("label", qname)])
         for edge in edges:
             g.add_edge(edge)
-        return g
+
+        if as_dot:
+            return write_dot(g)
+        else:
+            return g
 
     @_on_success
     def validate(self):
@@ -1375,6 +1412,12 @@ def _add(field):
                 resolved_packages.append(pkg.handle.to_dict())
             data["resolved_packages"] = resolved_packages
 
+        if _add("resolved_ephemerals"):
+            resolved_ephemerals = []
+            for ephemeral in (self._resolved_ephemerals or []):
+                resolved_ephemerals.append(str(ephemeral))
+            data["resolved_ephemerals"] = resolved_ephemerals
+
         if _add("serialize_version"):
             data["serialize_version"] = \
                 '.'.join(map(str, ResolvedContext.serialize_version))
@@ -1556,6 +1599,13 @@ def _print_version(value):
 
         r.package_caching = d.get("package_caching", True)
 
+        # -- SINCE SERIALIZE VERSION 4.6
+
+        r._resolved_ephemerals = []
+        for eph_str in d.get("resolved_ephemerals", []):
+            req = Requirement(eph_str)
+            r._resolved_ephemerals.append(req)
+
         # <END SERIALIZATION>
 
         # track context usage
@@ -1692,7 +1742,8 @@ def _get_pre_resolve_bindings(self):
                 "system": system,
                 "building": self.building,
                 "request": RequirementsBinding(self._package_requests),
-                "implicits": RequirementsBinding(self.implicit_packages)
+                "implicits": RequirementsBinding(self.implicit_packages),
+                "intersects": intersects
             }
 
         return self.pre_resolve_bindings
@@ -1701,23 +1752,29 @@ def _get_pre_resolve_bindings(self):
     def _execute(self, executor):
         # bind various info to the execution context
         resolved_pkgs = self.resolved_packages or []
+        ephemerals = self.resolved_ephemerals or []
+
         request_str = ' '.join(str(x) for x in self._package_requests)
         implicit_str = ' '.join(str(x) for x in self.implicit_packages)
         resolve_str = ' '.join(x.qualified_package_name for x in resolved_pkgs)
         package_paths_str = os.pathsep.join(self.package_paths)
+        req_timestamp_str = str(self.requested_timestamp or 0)
 
         header_comment(executor, "system setup")
 
         executor.setenv("REZ_USED", self.rez_path)
         executor.setenv("REZ_USED_VERSION", self.rez_version)
         executor.setenv("REZ_USED_TIMESTAMP", str(self.timestamp))
-        executor.setenv("REZ_USED_REQUESTED_TIMESTAMP",
-                        str(self.requested_timestamp or 0))
+        executor.setenv("REZ_USED_REQUESTED_TIMESTAMP", req_timestamp_str)
         executor.setenv("REZ_USED_REQUEST", request_str)
         executor.setenv("REZ_USED_IMPLICIT_PACKAGES", implicit_str)
         executor.setenv("REZ_USED_RESOLVE", resolve_str)
         executor.setenv("REZ_USED_PACKAGES_PATH", package_paths_str)
 
+        if ephemerals:
+            eph_resolve_str = ' '.join(str(x) for x in ephemerals)
+            executor.setenv("REZ_USED_EPH_RESOLVE", eph_resolve_str)
+
         if self.building:
             executor.setenv("REZ_BUILD_ENV", "1")
 
@@ -1733,11 +1790,12 @@ def _execute(self, executor):
             executor.setenv("REZ_RESOLVE_MODE", "latest")
 
         # binds objects such as 'request', which are accessible before a resolve
-        bindings = self._get_pre_resolve_bindings()
-        for k, v in bindings.items():
+        pre_resolve_bindings = self._get_pre_resolve_bindings()
+        for k, v in pre_resolve_bindings.items():
             executor.bind(k, v)
 
         executor.bind('resolve', VariantsBinding(resolved_pkgs))
+        executor.bind('ephemerals', EphemeralsBinding(ephemerals))
 
         #
         # -- apply each resolved package to the execution context
@@ -1751,7 +1809,6 @@ def _execute(self, executor):
 
         # retarget variant roots wrt package caching
         pkg_roots = {}
-
         if self.package_caching and \
                 config.cache_packages_path and \
                 config.read_package_cache:
@@ -1764,7 +1821,7 @@ def _execute(self, executor):
                         pkg_roots[pkg.name] = cached_root
 
         # set basic package variables and create per-package bindings
-        bindings = {}
+        pkg_bindings = {}
         for pkg in resolved_pkgs:
             minor_header_comment(executor, "variables for package %s" % pkg.qualified_name)
             prefix = "REZ_" + pkg.name.upper().replace('.', '_')
@@ -1786,10 +1843,12 @@ def _execute(self, executor):
             else:
                 executor.setenv(prefix + "_ROOT", pkg.root)
 
-            bindings[pkg.name] = dict(version=VersionBinding(pkg.version),
-                                      variant=VariantBinding(pkg))
+            pkg_bindings[pkg.name] = dict(
+                version=VersionBinding(pkg.version),
+                variant=VariantBinding(pkg)
+            )
 
-        # commands
+        # package commands
         for attr in ("pre_commands", "commands", "post_commands"):
             found = False
             for pkg in resolved_pkgs:
@@ -1801,14 +1860,13 @@ def _execute(self, executor):
                     header_comment(executor, attr)
 
                 minor_header_comment(executor, "%s from package %s" % (attr, pkg.qualified_name))
-                bindings_ = bindings[pkg.name]
-                executor.bind('this',       bindings_["variant"])
-                executor.bind("version",    bindings_["version"])
-                executor.bind('root',       pkg_roots.get(pkg.name, pkg.root))
-                executor.bind('base',       pkg.base)
+                pkg_bindings_ = pkg_bindings[pkg.name]
+                executor.bind('this', pkg_bindings_["variant"])
+                executor.bind("version", pkg_bindings_["version"])
+                executor.bind('root', pkg_roots.get(pkg.name, pkg.root))
+                executor.bind('base', pkg.base)
 
                 exc = None
-                trace = None
                 commands.set_package(pkg)
 
                 try:
@@ -1832,6 +1890,16 @@ def _execute(self, executor):
         for name in ("this", "version", "root", "base"):
             executor.unbind(name)
 
+        # set variables per ephemeral
+        # for eph '.foo-1.2' for eg, $REZ_EPH_FOO_REQUEST="1.2"
+        if ephemerals:
+            header_comment(executor, "ephemeral variables")
+
+            for eph_req in ephemerals:
+                uname = eph_req.name[1:].upper().replace('.', '_')
+                varname = "REZ_EPH_" + uname + "_REQUEST"
+                executor.setenv(varname, str(eph_req.range))
+
         header_comment(executor, "post system setup")
 
         # append suite paths based on suite visibility setting