From a3f8d899a9b0dc442412cdd89de3b754278b0cbd Mon Sep 17 00:00:00 2001 From: ajohns Date: Wed, 5 Dec 2018 15:38:50 +1100 Subject: [PATCH 1/8] -added build envvar REZ_BUILD_VARIANT_REQUIRES -addresses https://github.com/nerdvegas/rez/issues/416 --- src/rez/build_system.py | 3 +++ wiki/pages/Environment-Variables.md | 12 ++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/rez/build_system.py b/src/rez/build_system.py index 3a756fee9..3fa9d627f 100644 --- a/src/rez/build_system.py +++ b/src/rez/build_system.py @@ -170,11 +170,14 @@ def get_standard_vars(cls, context, variant, build_type, install, package = variant.parent + variant_requires = (variant.subpath or '').split(os.path.sep) + vars_ = { 'REZ_BUILD_ENV': 1, 'REZ_BUILD_PATH': build_path, 'REZ_BUILD_THREAD_COUNT': package.config.build_thread_count, 'REZ_BUILD_VARIANT_INDEX': variant.index or 0, + 'REZ_BUILD_VARIANT_REQUIRES': ' '.join(variant_requires), 'REZ_BUILD_PROJECT_VERSION': str(package.version), 'REZ_BUILD_PROJECT_NAME': package.name, 'REZ_BUILD_PROJECT_DESCRIPTION': (package.description or '').strip(), diff --git a/wiki/pages/Environment-Variables.md b/wiki/pages/Environment-Variables.md index 2b6b031b8..5933d9775 100644 --- a/wiki/pages/Environment-Variables.md +++ b/wiki/pages/Environment-Variables.md @@ -47,16 +47,20 @@ addition to those listed [here](#resolved-environment-variables). a *package.py* file). * **REZ_BUILD_PROJECT_NAME** - Name of the package being built. * **REZ_BUILD_PROJECT_VERSION** - Version of the package being built. -* **REZ_BUILD_REQUIRES** - The list of requirements for the build - comes from - the current package's *requires*, *build_requires* and *private_build_requires* - attributes, including the current variant's requirements. +* **REZ_BUILD_REQUIRES** - Space-separated list of requirements for the build - + comes from the current package's *requires*, *build_requires* and + *private_build_requires* attributes, including the current variant's requirements. * **REZ_BUILD_REQUIRES_UNVERSIONED** - Equivalent but unversioned list to *REZ_BUILD_REQUIRES*. * **REZ_BUILD_SOURCE_PATH** - Path containing the package.py file. * **REZ_BUILD_THREAD_COUNT** - Number of threads being used for the build. * **REZ_BUILD_TYPE** - One of *local* or *central*. Value is *central* if a release is occurring. -* **REZ_BUILD_VARIANT_INDEX** - Zero-based index of the variant currently being built. +* **REZ_BUILD_VARIANT_INDEX** - Zero-based index of the variant currently being + built. For non-varianted packages, this is "0". +* **REZ_BUILD_VARIANT_REQUIRES** - Space-separated list of runtime requirements + of the current variant. This does not include the common requirements as found + in *REZ_BUILD_REQUIRES*. For non-varianted builds, this is an empty string. ## System Environment Variables From c1c281330ea76494e58e469957c8261153dcab2d Mon Sep 17 00:00:00 2001 From: ajohns Date: Tue, 11 Dec 2018 09:13:25 +1100 Subject: [PATCH 2/8] -updated rez-depends to only include private_build_requires of the package being searched --- src/rez/cli/depends.py | 2 +- src/rez/package_search.py | 10 +++++++++- src/rez/utils/_version.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/rez/cli/depends.py b/src/rez/cli/depends.py index 8f6f9f994..2458a33ab 100644 --- a/src/rez/cli/depends.py +++ b/src/rez/cli/depends.py @@ -15,7 +15,7 @@ def setup_parser(parser, completions=False): help="Include build requirements") parser.add_argument( "-p", "--private-build-requires", action="store_true", - help="Include private build requirements") + help="Include private build requirements of PKG, if any") parser.add_argument( "-g", "--graph", action="store_true", help="display the dependency tree as an image") diff --git a/src/rez/package_search.py b/src/rez/package_search.py index 9d2c9233a..dd03602ed 100644 --- a/src/rez/package_search.py +++ b/src/rez/package_search.py @@ -37,6 +37,9 @@ def get_reverse_dependency_tree(package_name, depth=None, paths=None, depth (int): Tree depth limit, unlimited if None. paths (list of str): paths to search for packages, defaults to `config.packages_path`. + build_requires (bool): If True, includes packages' build_requires. + private_build_requires (bool): If True, include `package_name`'s + private_build_requires. Returns: A 2-tuple: @@ -71,7 +74,12 @@ def get_reverse_dependency_tree(package_name, depth=None, paths=None, requires = [] for variant in pkg.iter_variants(): - requires += variant.get_requires(build_requires, private_build_requires) + pbr = (private_build_requires and pkg.name == package_name) + + requires += variant.get_requires( + build_requires=build_requires, + private_build_requires=pbr + ) for req in requires: if not req.conflict: diff --git a/src/rez/utils/_version.py b/src/rez/utils/_version.py index 54816e680..7a4ecbbc4 100644 --- a/src/rez/utils/_version.py +++ b/src/rez/utils/_version.py @@ -1,7 +1,7 @@ # Update this value to version up Rez. Do not place anything else in this file. -_rez_version = "2.24.0" +_rez_version = "2.25.0" try: from rez.vendor.version.version import Version From b0f2c3bb411dbbc41f0c37dbe7f32dd029962d6c Mon Sep 17 00:00:00 2001 From: ajohns Date: Wed, 12 Dec 2018 14:29:44 +1100 Subject: [PATCH 3/8] -reevaluate package during build, for each variant -early-bound funcs can now access vars 'building', 'build_variant_index', 'build_variant_requires' --- src/rez/build_process_.py | 19 +++++++++- src/rez/build_system.py | 21 ++++++++--- src/rez/developer_package.py | 19 +++++++++- src/rez/packages_.py | 41 ++++++++++++++------ src/rez/rex.py | 4 +- src/rez/serialise.py | 54 +++++++++++++++++++++++---- src/rezplugins/build_system/cmake.py | 6 +-- src/rezplugins/build_system/custom.py | 19 ++++++---- 8 files changed, 141 insertions(+), 42 deletions(-) diff --git a/src/rez/build_process_.py b/src/rez/build_process_.py index cea7e0b73..02489184f 100644 --- a/src/rez/build_process_.py +++ b/src/rez/build_process_.py @@ -193,7 +193,24 @@ def visit_variants(self, func, variants=None, **kwargs): % (variant.index, self._n_of_m(variant))) continue - result = func(variant, **kwargs) + # Re-evaluate the variant, so that variables such as 'building' and + # 'build_variant_index' are set, and any early-bound package attribs + # are re-evaluated wrt these vars. + # + # Note: If you do something weird - like implement an early-bound + # 'variants' attrib that changes during a build - then behaviour is + # undefined. Don't do this. + # + re_evaluated_package = variant.parent.get_reevaluated({ + "building": True, + "build_variant_index": variant.index or 0, + "build_variant_requires": variant.variant_requires + }) + + re_evaluated_variant = re_evaluated_package.get_variant(variant.index) + + # visit the variant + result = func(re_evaluated_variant, **kwargs) results.append(result) num_visited += 1 diff --git a/src/rez/build_system.py b/src/rez/build_system.py index 3fa9d627f..12caf2f38 100644 --- a/src/rez/build_system.py +++ b/src/rez/build_system.py @@ -1,7 +1,9 @@ +import os.path + from rez.build_process_ import BuildType from rez.exceptions import BuildSystemError from rez.packages_ import get_developer_package -import os.path +from rez.backport.lru_cache import lru_cache def get_buildsys_types(): @@ -10,8 +12,16 @@ def get_buildsys_types(): return plugin_manager.get_plugins('build_system') +@lru_cache() def get_valid_build_systems(working_dir): - """Returns the build system classes that could build the source in given dir.""" + """Returns the build system classes that could build the source in given dir. + + Note: This function is cached because the 'custom' build system type causes + a package load (in order to get the 'build_command' package attribute). This + in turn causes early-bound attribs to be evaluated, and those could be + expensive (eg, a smart installer pkg that hits a website to get its valid + versions). + """ from rez.plugin_managers import plugin_manager clss = [] @@ -79,8 +89,8 @@ def __init__(self, working_dir, opts=None, package=None, working_dir: Directory to build source from. opts: argparse.Namespace object which may contain constructor params, as set by our bind_cli() classmethod. - package (`Package`): Package to build. If None, defaults to the - unbuilt ('developer') package in the working directory. + package (`DeveloperPackage`): Package to build. If None, defaults to + the package in the working directory. write_build_scripts: If True, create build scripts rather than perform the full build. The user can then run these scripts to place themselves into a build environment and invoke the build @@ -169,8 +179,7 @@ def get_standard_vars(cls, context, variant, build_type, install, from rez.config import config package = variant.parent - - variant_requires = (variant.subpath or '').split(os.path.sep) + variant_requires = map(str, variant.variant_requires) vars_ = { 'REZ_BUILD_ENV': 1, diff --git a/src/rez/developer_package.py b/src/rez/developer_package.py index 08ae7ce34..39b88ba46 100644 --- a/src/rez/developer_package.py +++ b/src/rez/developer_package.py @@ -1,6 +1,6 @@ from rez.config import config from rez.packages_ import Package -from rez.serialise import load_from_file, FileFormat +from rez.serialise import load_from_file, FileFormat, set_objects from rez.packages_ import create_package from rez.exceptions import PackageMetadataError, InvalidPackageError from rez.utils.system import add_sys_paths @@ -118,6 +118,23 @@ def visit(d): return package + def get_reevaluated(self, objects): + """Get a newly loaded and re-evaluated package. + + Values in `objects` are made available to early-bound package attributes. + For example, a re-evaluated package might return a different value for + an early-bound 'private_build_requires', depending on the variant + currently being built. + + Args: + objects (`dict`): Variables to expose to early-bound package attribs. + + Returns: + `DeveloperPackage`: New package. + """ + with set_objects(objects): + return self.from_path(self.root) + def _validate_includes(self): if not self.includes: return diff --git a/src/rez/packages_.py b/src/rez/packages_.py index 8abdcb73e..32364e56c 100644 --- a/src/rez/packages_.py +++ b/src/rez/packages_.py @@ -316,23 +316,31 @@ def parent(self): return self._parent + @property + def variant_requires(self): + """Get the subset of requirements specific to this variant. + + Returns: + List of `Requirement` objects. + """ + if self.index is None: + return [] + else: + return self.parent.variants[self.index] or [] + @property def requires(self): """Get variant requirements. This is a concatenation of the package requirements and those of this specific variant. - """ - try: - package_requires = self.parent.requires or [] - if self.index is None: - return package_requires - else: - variant_requires = self.parent.variants[self.index] or [] - return package_requires + variant_requires - except AttributeError as e: - reraise(e, ValueError) + Returns: + List of `Requirement` objects. + """ + return ( + (self.parent.requires or []) + self.variant_requires + ) def get_requires(self, build_requires=False, private_build_requires=False): """Get the requirements of the variant. @@ -433,9 +441,9 @@ def _repository_uids(self): return uids -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # resource acquisition functions -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ def iter_package_families(paths=None): """Iterate over package families, in no particular order. @@ -551,6 +559,15 @@ def get_package_from_string(txt, paths=None): def get_developer_package(path, format=None): + """Create a developer package. + + Args: + path (str): Path to dir containing package definition file. + format (str): Package definition file format, detected if None. + + Returns: + `DeveloperPackage`. + """ from rez.developer_package import DeveloperPackage return DeveloperPackage.from_path(path, format=format) diff --git a/src/rez/rex.py b/src/rez/rex.py index cfb3ecd00..78b215e8d 100644 --- a/src/rez/rex.py +++ b/src/rez/rex.py @@ -1141,8 +1141,8 @@ def compile_code(cls, code, filename=None, exec_namespace=None): code (str or SourceCode): The python code to compile. filename (str): File to associate with the code, will default to ''. - namespace (dict): Namespace to execute the code in. If None, the - code is not executed. + exec_namespace (dict): Namespace to execute the code in. If None, + the code is not executed. Returns: Compiled code object. diff --git a/src/rez/serialise.py b/src/rez/serialise.py index 5fcf31508..7a1b4bf64 100644 --- a/src/rez/serialise.py +++ b/src/rez/serialise.py @@ -1,6 +1,14 @@ """ Read and write data from file. File caching via a memcached server is supported. """ +from contextlib import contextmanager +from inspect import isfunction, ismodule, getargspec +from StringIO import StringIO +import sys +import os +import os.path +import threading + from rez.package_resources_ import package_rex_keys from rez.utils.scope import ScopeContext from rez.utils.sourcecode import SourceCode, early, late, include @@ -13,12 +21,6 @@ from rez.vendor.atomicwrites import atomic_write from rez.vendor.enum import Enum from rez.vendor import yaml -from contextlib import contextmanager -from inspect import isfunction, ismodule, getargspec -from StringIO import StringIO -import sys -import os -import os.path tmpdir_manager = TempDirs(config.tmpdir, prefix="rez_write_") @@ -138,6 +140,27 @@ def _load_file(filepath, format_, update_data_callback, original_filepath=None): return result +_set_objects = threading.local() + + +@contextmanager +def set_objects(objects): + """Set the objects made visible to early-bound attributes. + + For example, `objects` might be used to set a 'build_variant_index' var, so + that an early-bound 'private_build_requires' can change depending on the + currently-building variant. + + Args: + objects (dict): Variables to set. + """ + _set_objects.variables = objects + try: + yield + finally: + _set_objects.variables = {} + + def load_py(stream, filepath=None): """Load python-formatted data from a stream. @@ -185,7 +208,10 @@ def load_py(stream, filepath=None): class EarlyThis(object): - """The 'this' object for @early bound functions.""" + """The 'this' object for @early bound functions. + + Just exposes raw package data as object attributes. + """ def __init__(self, data): self._data = data @@ -237,9 +263,21 @@ def _process(value): argdefs=func.func_defaults, closure=func.func_closure) + # apply globals this = EarlyThis(data) - fn.func_globals.update({"this": this}) + fn.func_globals.update({ + "this": this, + "building": False, + # just some defaults to avoid not-defined errors in early- + # bound package attributes + "build_variant_index": 0, + "build_variant_requires": [] + }) + + fn.func_globals.update(getattr(_set_objects, "variables", {})) + + # execute the function with add_sys_paths(config.package_definition_build_python_paths): # this 'data' arg support isn't needed anymore, but I'm # supporting it til I know nobody is using it... diff --git a/src/rezplugins/build_system/cmake.py b/src/rezplugins/build_system/cmake.py index 9005dac33..3834b2055 100644 --- a/src/rezplugins/build_system/cmake.py +++ b/src/rezplugins/build_system/cmake.py @@ -92,10 +92,8 @@ def __init__(self, working_dir, opts=None, package=None, write_build_scripts=Fal child_build_args=child_build_args) self.settings = self.package.config.plugins.build_system.cmake - self.build_target = (opts and opts.build_target) or \ - self.settings.build_target - self.cmake_build_system = (opts and opts.build_system) or \ - self.settings.build_system + self.build_target = (opts and opts.build_target) or self.settings.build_target + self.cmake_build_system = (opts and opts.build_system) or self.settings.build_system if self.cmake_build_system == 'xcode' and platform_.name != 'osx': raise RezCMakeError("Generation of Xcode project only available " "on the OSX platform") diff --git a/src/rezplugins/build_system/custom.py b/src/rezplugins/build_system/custom.py index a0d6ec54a..8a8703c0a 100644 --- a/src/rezplugins/build_system/custom.py +++ b/src/rezplugins/build_system/custom.py @@ -1,18 +1,21 @@ """ Package-defined build command """ -from rez.build_system import BuildSystem -from rez.build_process_ import BuildType -from rez.packages_ import get_developer_package -from rez.exceptions import PackageMetadataError, BuildSystemError -from rez.utils.colorize import heading, Printer -from rez.utils.logging_ import print_warning from pipes import quote import functools import os.path import sys import os +from rez.build_system import BuildSystem +from rez.build_process_ import BuildType +from rez.packages_ import get_developer_package +from rez.resolved_context import ResolvedContext +from rez.exceptions import PackageMetadataError +from rez.utils.colorize import heading, Printer +from rez.utils.logging_ import print_warning +from rez.config import config + class CustomBuildSystem(BuildSystem): """This build system runs the 'build_command' defined in a package.py. @@ -37,7 +40,7 @@ def is_valid_root(cls, path): except PackageMetadataError: return False - return (getattr(package, "build_command", None) != None) + return (getattr(package, "build_command", None) is not None) def __init__(self, working_dir, opts=None, package=None, write_build_scripts=False, verbose=False, build_args=[], child_build_args=[]): @@ -180,7 +183,7 @@ def _add_build_actions(cls, executor, context, package, variant, def _FWD__spawn_build_shell(working_dir, build_path, variant_index, install, install_path=None): - # This spawns a shell that the user can run 'bez' in directly + # This spawns a shell that the user can run the build command in directly context = ResolvedContext.load(os.path.join(build_path, "build.rxt")) package = get_developer_package(working_dir) variant = package.get_variant(variant_index) From fe658658a0e2c4a39da42d515640a387c037bb8c Mon Sep 17 00:00:00 2001 From: ajohns Date: Wed, 12 Dec 2018 15:51:49 +1100 Subject: [PATCH 4/8] -updated late-bound funcs so they have access to new vars (build_variant_index etc) also --- src/rez/developer_package.py | 8 ++++- src/rez/packages_.py | 47 +++++++++++++++++++++++------ src/rez/serialise.py | 33 ++++++++++++-------- wiki/pages/Environment-Variables.md | 12 ++++---- 4 files changed, 72 insertions(+), 28 deletions(-) diff --git a/src/rez/developer_package.py b/src/rez/developer_package.py index 39b88ba46..6c4372994 100644 --- a/src/rez/developer_package.py +++ b/src/rez/developer_package.py @@ -133,7 +133,13 @@ def get_reevaluated(self, objects): `DeveloperPackage`: New package. """ with set_objects(objects): - return self.from_path(self.root) + package = self.from_path(self.root) + + # set same vars ('building' etc) for late-bound funcs (the previous + # set_objects context only sets them for early-bound funcs). + package.set_objects(objects) + + return package def _validate_includes(self): if not self.includes: diff --git a/src/rez/packages_.py b/src/rez/packages_.py index 32364e56c..d23550a2f 100644 --- a/src/rez/packages_.py +++ b/src/rez/packages_.py @@ -12,7 +12,7 @@ from rez.exceptions import PackageFamilyNotFoundError, ResourceError from rez.vendor.version.version import VersionRange from rez.vendor.version.requirement import VersionedObject -from rez.serialise import FileFormat +from rez.serialise import FileFormat, default_objects from rez.config import config import sys @@ -72,7 +72,9 @@ class PackageBaseResourceWrapper(PackageRepositoryResourceWrapper): def __init__(self, resource, context=None): super(PackageBaseResourceWrapper, self).__init__(resource) self.context = context - self._late_bindings = {} + + # cached results of late-bound funcs + self._late_binding_returnvalues = {} def set_context(self, context): self.context = context @@ -134,23 +136,27 @@ def print_info(self, buf=None, format_=FileFormat.yaml, def _wrap_forwarded(self, key, value): if isinstance(value, SourceCode) and value.late_binding: - value_ = self._late_bindings.get(key, KeyError) + # get cached return value if present + value_ = self._late_binding_returnvalues.get(key, KeyError) if value_ is KeyError: + # evaluate the late-bound function value_ = self._eval_late_binding(value) schema = self.late_bind_schemas.get(key) if schema is not None: value_ = schema.validate(value_) - self._late_bindings[key] = value_ + # cache result of late bound func + self._late_binding_returnvalues[key] = value_ return value_ else: return value def _eval_late_binding(self, sourcecode): - g = {} + g = default_objects.copy() + g.update(self._get_objects()) if self.context is None: g["in_context"] = lambda: False @@ -162,17 +168,22 @@ def _eval_late_binding(self, sourcecode): bindings = self.context._get_pre_resolve_bindings() g.update(bindings) - # note that what 'this' actually points to depends on whether the context - # is available or not. If not, then 'this' is a Package instance; if the - # context is available, it is a Variant instance. So for example, if - # in_context() is True, 'this' will have a 'root' attribute, but will + # Note that what 'this' actually points to depends on whether the context + # is available or not. If not, then 'this' is a DeveloperPackage instance; + # if the context is available, it is a Variant instance. So for example, + # if in_context() is True, 'this' will have a 'root' attribute, but will # not if in_context() is False. # g["this"] = self + # evaluate the late-bound function sourcecode.set_package(self) return sourcecode.exec_(globals_=g) + def _get_objects(self): + """Get variables to bind to late-bound funcs.""" + raise NotImplementedError + class Package(PackageBaseResourceWrapper): """A package. @@ -187,6 +198,10 @@ def __init__(self, resource, context=None): _check_class(resource, PackageResource) super(Package, self).__init__(resource, context) + # variables that are exposed to late-bound funcs on evaluation. Note + # that child variants are bound to these same variables. + self._late_binding_objects = {} + # arbitrary keys def __getattr__(self, name): if name in self.data: @@ -257,6 +272,17 @@ def get_variant(self, index=None): if variant.index == index: return variant + def set_objects(self, objects): + """Set bound variables for evaluation of late-bound attribs. + + Args: + objects (dict): Variables to bind. + """ + self._late_binding_objects = objects.copy() + + def _get_objects(self): + return self._late_binding_objects + class Variant(PackageBaseResourceWrapper): """A package variant. @@ -393,6 +419,9 @@ def install(self, path, dry_run=False, overrides=None): else: return Variant(resource) + def _get_objects(self): + return self.parent._get_objects() + class PackageSearchPath(object): """A list of package repositories. diff --git a/src/rez/serialise.py b/src/rez/serialise.py index 7a1b4bf64..e32269482 100644 --- a/src/rez/serialise.py +++ b/src/rez/serialise.py @@ -143,6 +143,25 @@ def _load_file(filepath, format_, update_data_callback, original_filepath=None): _set_objects = threading.local() +# Default variables to avoid not-defined errors in early/late- bound attribs +default_objects = { + "building": False, + "build_variant_index": 0, + "build_variant_requires": [] +} + + +def get_objects(): + """Get currently bound variables for evaluation of early-bound attribs. + + Returns: + dict. + """ + result = default_objects.copy() + result.update(getattr(_set_objects, "variables", {})) + return result + + @contextmanager def set_objects(objects): """Set the objects made visible to early-bound attributes. @@ -264,18 +283,8 @@ def _process(value): closure=func.func_closure) # apply globals - this = EarlyThis(data) - - fn.func_globals.update({ - "this": this, - "building": False, - # just some defaults to avoid not-defined errors in early- - # bound package attributes - "build_variant_index": 0, - "build_variant_requires": [] - }) - - fn.func_globals.update(getattr(_set_objects, "variables", {})) + fn.func_globals["this"] = EarlyThis(data) + fn.func_globals.update(get_objects()) # execute the function with add_sys_paths(config.package_definition_build_python_paths): diff --git a/wiki/pages/Environment-Variables.md b/wiki/pages/Environment-Variables.md index 5933d9775..4c3f8c06c 100644 --- a/wiki/pages/Environment-Variables.md +++ b/wiki/pages/Environment-Variables.md @@ -2,9 +2,9 @@ This chapter lists the environment variables that rez generates in certain circumstances, as well as environment variables that you can set which affect the operation of rez. -## Resolved Environment Variables +## Context Environment Variables -These are variables that rez generates within a resolved environment. +These are variables that rez generates within a resolved environment (a "context"). * **REZ_RXT_FILE** - Filepath of the current context (an rxt file). * **REZ_USED** - Path to rez installation that was used to resolve this environment. @@ -31,10 +31,10 @@ package name, *"(PKG)"* in the variables below is the uppercased package name. * **REZ_(PKG)_MINOR_VERSION** - The minor version of the package, or ''. * **REZ_(PKG)_PATCH_VERSION** - The patch version of the package, or ''. -## Resolved Build Environment Variables +## Build Environment Variables -These are variables that rez generates within a build environment; this is in -addition to those listed [here](#resolved-environment-variables). +These are variables that rez generates within a build environment, in addition +to those listed [here](#context-environment-variables). * **REZ_BUILD_ENV** - Always present in a build, has value 1. * **REZ_BUILD_INSTALL** - Has a value of 1 if an installation is taking place @@ -62,7 +62,7 @@ addition to those listed [here](#resolved-environment-variables). of the current variant. This does not include the common requirements as found in *REZ_BUILD_REQUIRES*. For non-varianted builds, this is an empty string. -## System Environment Variables +## Runtime Environment Variables These are environment variables that the user can set, which affect the operation of rez. From 2d7dee46743698447972705d188b812f459f8d1c Mon Sep 17 00:00:00 2001 From: ajohns Date: Thu, 13 Dec 2018 13:09:35 +1100 Subject: [PATCH 5/8] -removed buid_variant_index etc from late-bound vars, it is problematic (only valid during actual build) -updated wiki -fixed problem where early-bound building=true- based variants were installed, rather than the non-building-eval'd variants --- src/rez/build_process_.py | 18 +---- src/rez/developer_package.py | 16 ++--- src/rez/packages_.py | 44 ++++-------- src/rez/serialise.py | 2 +- src/rezplugins/build_process/local.py | 18 ++++- wiki/README.md | 4 +- wiki/pages/Package-Definition-Guide.md | 95 +++++++++++++++++++++++--- 7 files changed, 125 insertions(+), 72 deletions(-) diff --git a/src/rez/build_process_.py b/src/rez/build_process_.py index 02489184f..fa6b68ca3 100644 --- a/src/rez/build_process_.py +++ b/src/rez/build_process_.py @@ -193,24 +193,8 @@ def visit_variants(self, func, variants=None, **kwargs): % (variant.index, self._n_of_m(variant))) continue - # Re-evaluate the variant, so that variables such as 'building' and - # 'build_variant_index' are set, and any early-bound package attribs - # are re-evaluated wrt these vars. - # - # Note: If you do something weird - like implement an early-bound - # 'variants' attrib that changes during a build - then behaviour is - # undefined. Don't do this. - # - re_evaluated_package = variant.parent.get_reevaluated({ - "building": True, - "build_variant_index": variant.index or 0, - "build_variant_requires": variant.variant_requires - }) - - re_evaluated_variant = re_evaluated_package.get_variant(variant.index) - # visit the variant - result = func(re_evaluated_variant, **kwargs) + result = func(variant, **kwargs) results.append(result) num_visited += 1 diff --git a/src/rez/developer_package.py b/src/rez/developer_package.py index 6c4372994..ea470338e 100644 --- a/src/rez/developer_package.py +++ b/src/rez/developer_package.py @@ -121,10 +121,10 @@ def visit(d): def get_reevaluated(self, objects): """Get a newly loaded and re-evaluated package. - Values in `objects` are made available to early-bound package attributes. - For example, a re-evaluated package might return a different value for - an early-bound 'private_build_requires', depending on the variant - currently being built. + Values in `objects` are made available to early/late bound package + attributes. For example, a re-evaluated package might return a different + value for an early-bound 'private_build_requires', depending on the + variant currently being built. Args: objects (`dict`): Variables to expose to early-bound package attribs. @@ -133,13 +133,7 @@ def get_reevaluated(self, objects): `DeveloperPackage`: New package. """ with set_objects(objects): - package = self.from_path(self.root) - - # set same vars ('building' etc) for late-bound funcs (the previous - # set_objects context only sets them for early-bound funcs). - package.set_objects(objects) - - return package + return self.from_path(self.root) def _validate_includes(self): if not self.includes: diff --git a/src/rez/packages_.py b/src/rez/packages_.py index d23550a2f..f25012428 100644 --- a/src/rez/packages_.py +++ b/src/rez/packages_.py @@ -12,7 +12,7 @@ from rez.exceptions import PackageFamilyNotFoundError, ResourceError from rez.vendor.version.version import VersionRange from rez.vendor.version.requirement import VersionedObject -from rez.serialise import FileFormat, default_objects +from rez.serialise import FileFormat from rez.config import config import sys @@ -155,8 +155,7 @@ def _wrap_forwarded(self, key, value): return value def _eval_late_binding(self, sourcecode): - g = default_objects.copy() - g.update(self._get_objects()) + g = {} if self.context is None: g["in_context"] = lambda: False @@ -168,11 +167,8 @@ def _eval_late_binding(self, sourcecode): bindings = self.context._get_pre_resolve_bindings() g.update(bindings) - # Note that what 'this' actually points to depends on whether the context - # is available or not. If not, then 'this' is a DeveloperPackage instance; - # if the context is available, it is a Variant instance. So for example, - # if in_context() is True, 'this' will have a 'root' attribute, but will - # not if in_context() is False. + # Note that 'this' could be a `Package` or `Variant` instance. This is + # intentional; it just depends on how the package is accessed. # g["this"] = self @@ -180,10 +176,6 @@ def _eval_late_binding(self, sourcecode): sourcecode.set_package(self) return sourcecode.exec_(globals_=g) - def _get_objects(self): - """Get variables to bind to late-bound funcs.""" - raise NotImplementedError - class Package(PackageBaseResourceWrapper): """A package. @@ -194,14 +186,16 @@ class Package(PackageBaseResourceWrapper): """ keys = schema_keys(package_schema) + # This is to allow for a simple check like 'this.is_package' in late-bound + # funcs, where 'this' may be a package or variant. + # + is_package = True + is_variant = False + def __init__(self, resource, context=None): _check_class(resource, PackageResource) super(Package, self).__init__(resource, context) - # variables that are exposed to late-bound funcs on evaluation. Note - # that child variants are bound to these same variables. - self._late_binding_objects = {} - # arbitrary keys def __getattr__(self, name): if name in self.data: @@ -272,17 +266,6 @@ def get_variant(self, index=None): if variant.index == index: return variant - def set_objects(self, objects): - """Set bound variables for evaluation of late-bound attribs. - - Args: - objects (dict): Variables to bind. - """ - self._late_binding_objects = objects.copy() - - def _get_objects(self): - return self._late_binding_objects - class Variant(PackageBaseResourceWrapper): """A package variant. @@ -294,6 +277,10 @@ class Variant(PackageBaseResourceWrapper): keys = schema_keys(variant_schema) keys.update(["index", "root", "subpath"]) + # See comment in `Package` + is_package = False + is_variant = True + def __init__(self, resource, context=None, parent=None): _check_class(resource, VariantResource) super(Variant, self).__init__(resource, context) @@ -419,9 +406,6 @@ def install(self, path, dry_run=False, overrides=None): else: return Variant(resource) - def _get_objects(self): - return self.parent._get_objects() - class PackageSearchPath(object): """A list of package repositories. diff --git a/src/rez/serialise.py b/src/rez/serialise.py index e32269482..8ec62977b 100644 --- a/src/rez/serialise.py +++ b/src/rez/serialise.py @@ -143,7 +143,7 @@ def _load_file(filepath, format_, update_data_callback, original_filepath=None): _set_objects = threading.local() -# Default variables to avoid not-defined errors in early/late- bound attribs +# Default variables to avoid not-defined errors in early-bound attribs default_objects = { "building": False, "build_variant_index": 0, diff --git a/src/rezplugins/build_process/local.py b/src/rezplugins/build_process/local.py index 36189120c..22ece59fd 100644 --- a/src/rezplugins/build_process/local.py +++ b/src/rezplugins/build_process/local.py @@ -119,9 +119,25 @@ def _build_variant_base(self, variant, build_type, install_path=None, if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) + # Re-evaluate the variant, so that variables such as 'building' and + # 'build_variant_index' are set, and any early-bound package attribs + # are re-evaluated wrt these vars. This is done so that attribs such as + # 'requires' can change depending on whether a build is occurring or not. + # + # Note that this re-evaluated variant is ONLY used here, for the purposes + # of creating the build context. The variant that is actually installed + # is the one evaluated where 'building' is False. + # + re_evaluated_package = variant.parent.get_reevaluated({ + "building": True, + "build_variant_index": variant.index or 0, + "build_variant_requires": variant.variant_requires + }) + re_evaluated_variant = re_evaluated_package.get_variant(variant.index) + # create build environment context, rxt_filepath = self.create_build_context( - variant=variant, + variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) diff --git a/wiki/README.md b/wiki/README.md index eb118195e..76063ebe6 100644 --- a/wiki/README.md +++ b/wiki/README.md @@ -5,4 +5,6 @@ found [here](https://github.com/nerdvegas/rez/wiki). You should include relevant wiki updates with your code PRs. -To update the wiki, make your changes here, then run ./update-wiki.sh. +To update the wiki, make your changes here, then run ./update-wiki.sh. The +current repository status is irrelevant - you don't have to have anything +committed nor pushed in order to update the wiki repository. diff --git a/wiki/pages/Package-Definition-Guide.md b/wiki/pages/Package-Definition-Guide.md index ea5f54df1..7b3c91475 100644 --- a/wiki/pages/Package-Definition-Guide.md +++ b/wiki/pages/Package-Definition-Guide.md @@ -108,6 +108,38 @@ define an arbitrary function earlier in the python source. You can always use a two as well - an early binding function can call an arbitrary function defined at the bottom of your definition file. +#### Available Objects + +Following is the list of objects that are available during early evaluation. + +* *building* - see [building](Package-Commands#building); +* *build_variant_index* - the index of the variant currently being built. This is only relevant if + `building` is True. +* *build_variant_requires* - the subset of package requirements specific to the variant + currently being built. This is a list of `PackageRequest` objects. This is only relevant if + `building` is True. +* *this* - the current package, as described previously. + +Be aware that early-bound functions are actually evaluated multiple times during a build - once +pre-build, and once per variant, during its build. This is necessary in order for early-bound +functions to change their return value based on variables like `build_variant_index`. Note that the +*pre-build* evaluated value is the one set into the installed package, and in this case, `building` +is False. + +An example of where you'd need to be aware of this is if you wanted the `requires` field to include +a certain package at runtime only (ie, not present during the package build). In this case, `requires` +might look like so: + + @early() + def requires(): + if building: + return ["python-2"] + else: + return ["runtimeonly-1.2", "python-2"] + +> [[media/icons/warning.png]] You **must** ensure that your early-bound function returns the value +> you want to see in the installed package, when `building` is False. + ### Late Binding Functions Late binding functions stay as functions in the installed package definition, and are only evaluated @@ -194,23 +226,64 @@ late binding *tool* attribute below: Here the *request* object is being checked to see if the *maya* package was requested in the current env; if it was, a maya-specific tool *maya-edit* is added to the tool list. +> [[media/icons/warning.png]] Always ensure your late binding function returns a sensible +> value regardless of whether *in_context* is True or False. Otherwise, simply trying to +> query the package attributes (using *rez-search* for example) may cause errors. + +#### Available Objects + Following is the list of objects that are available during late evaluation, if *in_context* -is true: +is *True*: -* **context** - the *ResolvedContext* instance this package belongs to; -* **system** - see [system](Package-Commands#system); -* **building** - see [building](Package-Commands#building); -* **request** - see [request](Package-Commands#request); -* **implicits** - see [implicits](Package-Commands#implicits). +* *context* - the *ResolvedContext* instance this package belongs to; +* *system* - see [system](Package-Commands#system); +* *building* - see [building](Package-Commands#building); +* *request* - see [request](Package-Commands#request); +* *implicits* - see [implicits](Package-Commands#implicits). The following objects are available in *all* cases: -* **this** - the current package; -* **in_context** - the *in_context* function itself. +* *this* - the current package/variant (see note below); +* *in_context* - the *in_context* function itself. -> [[media/icons/warning.png]] Always ensure your late binding function returns a sensible -> value regardless of whether *in_context* is true or false. Otherwise, simply trying to -> query the package attributes (using *rez-search* for example) may cause errors. +> [[media/icons/warning.png]] The *this* object may be either a package or a variant, +> depending on the situation. For example, if *in_context* is True, then *this* is a +> variant, because variants are the objects present in a resolved context. On the other +> hand, if a package is accessed via API (for example, by using the *rez-search* tool), +> then *this* may be a package. The difference matters, because variants have some +> attributes that packages don't - notably, *root* and *index*. Use the properties +> `this.is_package` and `this.is_variant` to distinguish the case if needed. + +#### Example - Late Bound build_requires + +Here is an example of a package.py with a late-bound `build_requires` field: + + name = "maya_thing" + + version = "1.0.0" + + variants = [ + ["maya-2017"], + ["maya-2018"] + ] + + @late() + def build_requires(): + if this.is_package: + return [] + elif this.index == 0: + return ["maya_2017_build_utils"] + else: + return ["maya_2018_build_utils"] + +Note the check for `this.is_package`. This is necessary, otherwise the evaluation would +fail in some circumstances. Specifically, if someone ran the following command, the `this` +field would actually be a `Package` instance, which doesn't have an `index`: + + ]$ rez-search maya_thing --type package --format '{build_requires}' + +In this case, `build_requires` is somewhat nonsensical (there is no common build requirement +for both variants here), but something needs to be returned nonetheless. ## Sharing Code Across Package Definition Files From e5405604388c1e412ed5481fb79f1252d04f4083 Mon Sep 17 00:00:00 2001 From: ajohns Date: Thu, 13 Dec 2018 13:29:59 +1100 Subject: [PATCH 6/8] addresses https://github.com/nerdvegas/rez/issues/433 --- src/rez/serialise.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/rez/serialise.py b/src/rez/serialise.py index 8ec62977b..c2d933f35 100644 --- a/src/rez/serialise.py +++ b/src/rez/serialise.py @@ -189,6 +189,11 @@ def load_py(stream, filepath=None): Returns: dict. """ + with add_sys_paths(config.package_definition_build_python_paths): + return _load_py(stream, filepath=filepath) + + +def _load_py(stream, filepath=None): scopes = ScopeContext() g = dict(scope=scopes, @@ -287,19 +292,18 @@ def _process(value): fn.func_globals.update(get_objects()) # execute the function - with add_sys_paths(config.package_definition_build_python_paths): + spec = getargspec(func) + args = spec.args or [] + if len(args) not in (0, 1): + raise ResourceError("@early decorated function must " + "take zero or one args only") + if args: # this 'data' arg support isn't needed anymore, but I'm # supporting it til I know nobody is using it... # - spec = getargspec(func) - args = spec.args or [] - if len(args) not in (0, 1): - raise ResourceError("@early decorated function must " - "take zero or one args only") - if args: - value_ = fn(data) - else: - value_ = fn() + value_ = fn(data) + else: + value_ = fn() # process again in case this is a function returning a function return _process(value_) From 933744d6f51be747c34f05089e9d59bdbeba2e3b Mon Sep 17 00:00:00 2001 From: ajohns Date: Thu, 13 Dec 2018 14:10:49 +1100 Subject: [PATCH 7/8] -added rez-cp --follow-symlinks arg -addresses https://github.com/nerdvegas/rez/issues/547 --- src/rez/cli/cp.py | 7 ++++++- src/rez/package_copy.py | 17 +++++++++++++---- src/rez/utils/filesystem.py | 4 ++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/rez/cli/cp.py b/src/rez/cli/cp.py index 5f5b32e17..ee0aad3a3 100644 --- a/src/rez/cli/cp.py +++ b/src/rez/cli/cp.py @@ -27,7 +27,11 @@ def setup_parser(parser, completions=False): help="overwrite existing package/variants") parser.add_argument( "-s", "--shallow", action="store_true", - help="perform a shallow copy (symlink directories)") + help="perform a shallow copy (symlinks topmost directories)") + parser.add_argument( + "--follow-symlinks", action="store_true", + help="follow symlinks when copying package payload, rather than copying " + "the symlinks themselves.") parser.add_argument( "-k", "--keep-timestamp", action="store_true", help="keep timestamp of source package. Note that this is ignored if " @@ -134,6 +138,7 @@ def command(opts, parser, extra_arg_groups=None): variants=variants, overwrite=opts.overwrite, shallow=opts.shallow, + follow_symlinks=opts.follow_symlinks, keep_timestamp=opts.keep_timestamp, force=opts.force, verbose=opts.verbose, diff --git a/src/rez/package_copy.py b/src/rez/package_copy.py index dc8054a17..4f48fc610 100644 --- a/src/rez/package_copy.py +++ b/src/rez/package_copy.py @@ -1,3 +1,4 @@ +from functools import partial import os.path import time @@ -11,8 +12,8 @@ def copy_package(package, dest_repository, variants=None, shallow=False, dest_name=None, dest_version=None, overwrite=False, force=False, - dry_run=False, keep_timestamp=False, skip_payload=False, - overrides=None, verbose=False): + follow_symlinks=False, dry_run=False, keep_timestamp=False, + skip_payload=False, overrides=None, verbose=False): """Copy a package from one package repository to another. This copies the package definition and payload. The package can also be @@ -58,6 +59,8 @@ def copy_package(package, dest_repository, variants=None, shallow=False, force (bool): Copy the package regardless of its relocatable attribute. Use at your own risk (there is no guarantee the resulting package will be functional). + follow_symlinks (bool): Follow symlinks when copying package payload, + rather than copying the symlinks themselves. keep_timestamp (bool): By default, a newly copied package will get a new timestamp (because that's when it was added to the target repo). By setting this option to True, the original package's timestamp @@ -182,6 +185,7 @@ def finalize(): src_variant=src_variant, dest_pkg_repo=dest_pkg_repo, shallow=shallow, + follow_symlinks=follow_symlinks, overrides=overrides ) @@ -207,7 +211,7 @@ def finalize(): def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, - overrides=None): + follow_symlinks=False, overrides=None): # Get payload path of source variant. For some types (eg from a "memory" # type repo) there may not be a root. # @@ -241,7 +245,12 @@ def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False, if shallow: maybe_symlink = replacing_symlink else: - maybe_symlink = replacing_copy + maybe_symlink = partial( + replacing_copy, + copytree_kwargs={ + "symlinks": (not follow_symlinks) + } + ) if src_variant.subpath: # symlink/copy the last install dir to the variant root diff --git a/src/rez/utils/filesystem.py b/src/rez/utils/filesystem.py index c3b04954a..d877a5700 100644 --- a/src/rez/utils/filesystem.py +++ b/src/rez/utils/filesystem.py @@ -122,7 +122,7 @@ def replacing_symlink(source, link_name): replace_file_or_dir(link_name, tmp_link_name) -def replacing_copy(src, dest): +def replacing_copy(src, dest, copytree_kwargs=None): """Perform copy that overwrites any existing target. Will copy/copytree `src` to `dest`, and will remove `dest` if it exists, @@ -133,7 +133,7 @@ def replacing_copy(src, dest): """ with make_tmp_name(dest) as tmp_dest: if os.path.isdir(src) and not os.path.islink(src): - shutil.copytree(src, tmp_dest) + shutil.copytree(src, tmp_dest, **(copytree_kwargs or {})) else: shutil.copy2(src, tmp_dest) From 98bcc464e7da7b158e819535b2fe2191011c6b4c Mon Sep 17 00:00:00 2001 From: ajohns Date: Thu, 13 Dec 2018 14:21:39 +1100 Subject: [PATCH 8/8] docstring fix --- src/rez/developer_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rez/developer_package.py b/src/rez/developer_package.py index ea470338e..7b7cd65c3 100644 --- a/src/rez/developer_package.py +++ b/src/rez/developer_package.py @@ -121,7 +121,7 @@ def visit(d): def get_reevaluated(self, objects): """Get a newly loaded and re-evaluated package. - Values in `objects` are made available to early/late bound package + Values in `objects` are made available to early-bound package attributes. For example, a re-evaluated package might return a different value for an early-bound 'private_build_requires', depending on the variant currently being built.