From 37688ab717401dc396d4aedc6168dd726f24e6e5 Mon Sep 17 00:00:00 2001 From: ajohns Date: Tue, 21 Jan 2020 10:54:31 +1100 Subject: [PATCH] getting ready to fully deprecate old an annoying trailing-underscore sourcefiles --- src/rez/bind/_pymodule.py | 2 +- src/rez/bind/arch.py | 2 +- src/rez/bind/cmake.py | 2 +- src/rez/bind/gcc.py | 2 +- src/rez/bind/hello_world.py | 2 +- src/rez/bind/os.py | 2 +- src/rez/bind/platform.py | 2 +- src/rez/bind/python.py | 2 +- src/rez/bind/rez.py | 2 +- src/rez/bind/rezgui.py | 2 +- src/rez/build_process.py | 454 ++++++++++ src/rez/build_process_.py | 458 +--------- src/rez/build_system.py | 4 +- src/rez/cli/_complete_util.py | 4 +- src/rez/cli/build.py | 6 +- src/rez/cli/cp.py | 2 +- src/rez/cli/diff.py | 2 +- src/rez/cli/memcache.py | 2 +- src/rez/cli/release.py | 2 +- src/rez/cli/view.py | 2 +- src/rez/cli/yaml2py.py | 2 +- src/rez/developer_package.py | 3 +- src/rez/package_filter.py | 2 +- src/rez/package_help.py | 2 +- src/rez/package_maker.py | 249 ++++++ src/rez/package_maker__.py | 253 +----- src/rez/package_py_utils.py | 4 +- src/rez/package_resources.py | 528 ++++++++++++ src/rez/package_resources_.py | 530 +----------- src/rez/package_search.py | 2 +- src/rez/package_serialise.py | 2 +- src/rez/package_test.py | 2 +- src/rez/packages.py | 792 +++++++++++++++++ src/rez/packages_.py | 796 +----------------- src/rez/pip.py | 4 +- src/rez/release_hook.py | 2 +- src/rez/release_vcs.py | 2 +- src/rez/resolved_context.py | 2 +- src/rez/resolver.py | 2 +- src/rez/serialise.py | 2 +- src/rez/solver.py | 2 +- src/rez/status.py | 2 +- src/rez/tests/test_build.py | 2 +- src/rez/tests/test_completion.py | 2 +- src/rez/tests/test_config.py | 2 +- src/rez/tests/test_copy_package.py | 4 +- src/rez/tests/test_imports.py | 8 +- src/rez/tests/test_packages.py | 4 +- src/rez/tests/test_release.py | 4 +- src/rez/utils/_version.py | 2 +- src/rez/utils/diff_packages.py | 2 +- src/rez/utils/sourcecode.py | 2 +- src/rez/wrapper.py | 2 +- src/rezgui/objects/LoadPackagesThread.py | 2 +- src/rezgui/widgets/ContextTableWidget.py | 2 +- src/rezgui/widgets/PackageLineEdit.py | 2 +- src/rezgui/widgets/PackageTabWidget.py | 2 +- src/rezgui/widgets/PackageVersionsTable.py | 2 +- src/rezgui/widgets/VariantCellWidget.py | 2 +- src/rezgui/widgets/VariantSummaryWidget.py | 2 +- src/rezgui/widgets/VariantVersionsTable.py | 2 +- src/rezgui/widgets/VariantsList.py | 2 +- src/rezplugins/build_process/local.py | 2 +- src/rezplugins/build_process/remote.py | 2 +- src/rezplugins/build_system/bez.py | 4 +- src/rezplugins/build_system/cmake.py | 4 +- src/rezplugins/build_system/custom.py | 4 +- .../package_repository/filesystem.py | 2 +- src/rezplugins/package_repository/memory.py | 2 +- 69 files changed, 2122 insertions(+), 2091 deletions(-) create mode 100644 src/rez/build_process.py create mode 100644 src/rez/package_maker.py create mode 100644 src/rez/package_resources.py create mode 100644 src/rez/packages.py diff --git a/src/rez/bind/_pymodule.py b/src/rez/bind/_pymodule.py index b80926848..b3af38de3 100644 --- a/src/rez/bind/_pymodule.py +++ b/src/rez/bind/_pymodule.py @@ -9,7 +9,7 @@ from rez.bind._utils import check_version, find_exe, make_dirs, \ get_version_in_python, run_python_command, log -from rez.package_maker__ import make_package +from rez.package_maker import make_package from rez.exceptions import RezBindError from rez.system import system from rez.utils.logging_ import print_warning diff --git a/src/rez/bind/arch.py b/src/rez/bind/arch.py index 6c83f4d7b..a9d8f0822 100644 --- a/src/rez/bind/arch.py +++ b/src/rez/bind/arch.py @@ -2,7 +2,7 @@ Creates the system architecture package. """ from __future__ import absolute_import -from rez.package_maker__ import make_package +from rez.package_maker import make_package from rez.vendor.version.version import Version from rez.bind._utils import check_version from rez.system import system diff --git a/src/rez/bind/cmake.py b/src/rez/bind/cmake.py index 781fecc1d..5e352c57c 100644 --- a/src/rez/bind/cmake.py +++ b/src/rez/bind/cmake.py @@ -2,7 +2,7 @@ Binds a cmake executable as a rez package. """ from __future__ import absolute_import -from rez.package_maker__ import make_package +from rez.package_maker import make_package from rez.bind._utils import check_version, find_exe, extract_version, make_dirs from rez.utils.platform_ import platform_ from rez.system import system diff --git a/src/rez/bind/gcc.py b/src/rez/bind/gcc.py index aff5da232..98ba32520 100644 --- a/src/rez/bind/gcc.py +++ b/src/rez/bind/gcc.py @@ -1,6 +1,6 @@ from __future__ import absolute_import from rez.bind._utils import find_exe, extract_version, make_dirs, log -from rez.package_maker__ import make_package +from rez.package_maker import make_package from rez.utils.lint_helper import env from rez.utils.platform_ import platform_ import os.path diff --git a/src/rez/bind/hello_world.py b/src/rez/bind/hello_world.py index e46041a44..3237e7d4e 100644 --- a/src/rez/bind/hello_world.py +++ b/src/rez/bind/hello_world.py @@ -7,7 +7,7 @@ """ from __future__ import absolute_import, print_function -from rez.package_maker__ import make_package +from rez.package_maker import make_package from rez.vendor.version.version import Version from rez.utils.lint_helper import env from rez.utils.execution import create_executable_script, ExecutableScriptMode diff --git a/src/rez/bind/os.py b/src/rez/bind/os.py index 2d5151116..8ddb98ecc 100644 --- a/src/rez/bind/os.py +++ b/src/rez/bind/os.py @@ -2,7 +2,7 @@ Creates the operating system package. """ from __future__ import absolute_import -from rez.package_maker__ import make_package +from rez.package_maker import make_package from rez.vendor.version.version import Version from rez.bind._utils import check_version from rez.system import system diff --git a/src/rez/bind/platform.py b/src/rez/bind/platform.py index 2ad6239d6..298d9c7be 100644 --- a/src/rez/bind/platform.py +++ b/src/rez/bind/platform.py @@ -2,7 +2,7 @@ Creates the system platform package. """ from __future__ import absolute_import -from rez.package_maker__ import make_package +from rez.package_maker import make_package from rez.vendor.version.version import Version from rez.bind._utils import check_version from rez.system import system diff --git a/src/rez/bind/python.py b/src/rez/bind/python.py index b0c17d6ed..b2107f941 100644 --- a/src/rez/bind/python.py +++ b/src/rez/bind/python.py @@ -4,7 +4,7 @@ from __future__ import absolute_import from rez.bind._utils import check_version, find_exe, extract_version, \ make_dirs, log, run_python_command -from rez.package_maker__ import make_package +from rez.package_maker import make_package from rez.system import system from rez.utils.lint_helper import env from rez.utils.platform_ import platform_ diff --git a/src/rez/bind/rez.py b/src/rez/bind/rez.py index 6dc734575..329c85b7c 100644 --- a/src/rez/bind/rez.py +++ b/src/rez/bind/rez.py @@ -3,7 +3,7 @@ """ from __future__ import absolute_import import rez -from rez.package_maker__ import make_package +from rez.package_maker import make_package from rez.bind._utils import check_version from rez.system import system from rez.utils.lint_helper import env diff --git a/src/rez/bind/rezgui.py b/src/rez/bind/rezgui.py index 1c40a3525..090db37b9 100644 --- a/src/rez/bind/rezgui.py +++ b/src/rez/bind/rezgui.py @@ -3,7 +3,7 @@ """ from __future__ import absolute_import import rez -from rez.package_maker__ import make_package +from rez.package_maker import make_package from rez.bind._utils import check_version, make_dirs from rez.system import system from rez.vendor.version.version import Version diff --git a/src/rez/build_process.py b/src/rez/build_process.py new file mode 100644 index 000000000..036aa0762 --- /dev/null +++ b/src/rez/build_process.py @@ -0,0 +1,454 @@ +from __future__ import print_function + +from rez.packages import iter_packages +from rez.exceptions import BuildProcessError, BuildContextResolveError, \ + ReleaseHookCancellingError, RezError, ReleaseError, BuildError, \ + ReleaseVCSError +from rez.utils.logging_ import print_warning +from rez.utils.colorize import heading, Printer +from rez.resolved_context import ResolvedContext +from rez.release_hook import create_release_hooks +from rez.resolver import ResolverStatus +from rez.config import config +from rez.vendor.enum import Enum +from contextlib import contextmanager +from pipes import quote +import getpass +import os.path +import sys + + +debug_print = config.debug_printer("package_release") + + +def get_build_process_types(): + """Returns the available build process implementations.""" + from rez.plugin_managers import plugin_manager + return plugin_manager.get_plugins('build_process') + + +def create_build_process(process_type, working_dir, build_system, package=None, + vcs=None, ensure_latest=True, skip_repo_errors=False, + ignore_existing_tag=False, verbose=False, quiet=False): + """Create a `BuildProcess` instance.""" + from rez.plugin_managers import plugin_manager + process_types = get_build_process_types() + if process_type not in process_types: + raise BuildProcessError("Unknown build process: %r" % process_type) + + cls = plugin_manager.get_plugin_class('build_process', process_type) + + return cls(working_dir, # ignored (deprecated) + build_system, + package=package, # ignored (deprecated) + vcs=vcs, + ensure_latest=ensure_latest, + skip_repo_errors=skip_repo_errors, + ignore_existing_tag=ignore_existing_tag, + verbose=verbose, + quiet=quiet) + + +class BuildType(Enum): + """ Enum to represent the type of build.""" + local = 0 + central = 1 + + +class BuildProcess(object): + """A BuildProcess builds and possibly releases a package. + + A build process iterates over the variants of a package, creates the + correct build environment for each variant, builds that variant using a + build system (or possibly creates a script so the user can do that + independently), and then possibly releases the package with the nominated + VCS. This is an abstract base class, you should use a BuildProcess + subclass. + """ + @classmethod + def name(cls): + raise NotImplementedError + + def __init__(self, working_dir, build_system, package=None, vcs=None, + ensure_latest=True, skip_repo_errors=False, + ignore_existing_tag=False, verbose=False, quiet=False): + """Create a BuildProcess. + + Args: + working_dir (DEPRECATED): Ignored. + build_system (`BuildSystem`): Build system used to build the package. + package (DEPRECATED): Ignored. + vcs (`ReleaseVCS`): Version control system to use for the release + process. + ensure_latest: If True, do not allow the release process to occur + if an newer versioned package is already released. + skip_repo_errors: If True, proceed with the release even when errors + occur. BE CAREFUL using this option, it is here in case a package + needs to be released urgently even though there is some problem + with reading or writing the repository. + ignore_existing_tag: Perform the release even if the repository is + already tagged at the current version. If the config setting + plugins.release_vcs.check_tag is False, this has no effect. + verbose (bool): Verbose mode. + quiet (bool): Quiet mode (overrides `verbose`). + """ + self.verbose = verbose and not quiet + self.quiet = quiet + self.build_system = build_system + self.vcs = vcs + self.ensure_latest = ensure_latest + self.skip_repo_errors = skip_repo_errors + self.ignore_existing_tag = ignore_existing_tag + + if vcs and vcs.pkg_root != self.working_dir: + raise BuildProcessError( + "Build process was instantiated with a mismatched VCS instance") + + self.build_path = os.path.join(self.working_dir, + self.package.config.build_directory) + + @property + def package(self): + return self.build_system.package + + @property + def working_dir(self): + return self.build_system.working_dir + + def build(self, install_path=None, clean=False, install=False, variants=None): + """Perform the build process. + + Iterates over the package's variants, resolves the environment for + each, and runs the build system within each resolved environment. + + Args: + install_path (str): The package repository path to install the + package to, if installing. If None, defaults to + `config.local_packages_path`. + clean (bool): If True, clear any previous build first. Otherwise, + rebuild over the top of a previous build. + install (bool): If True, install the build. + variants (list of int): Indexes of variants to build, all if None. + + Raises: + `BuildError`: If the build failed. + + Returns: + int: Number of variants successfully built. + """ + raise NotImplementedError + + def release(self, release_message=None, variants=None): + """Perform the release process. + + Iterates over the package's variants, building and installing each into + the release path determined by `config.release_packages_path`. + + Args: + release_message (str): Message to associate with the release. + variants (list of int): Indexes of variants to release, all if None. + + Raises: + `ReleaseError`: If the release failed. + + Returns: + int: Number of variants successfully released. + """ + raise NotImplementedError + + def get_changelog(self): + """Get the changelog since last package release. + + Returns: + str: Changelog. + """ + raise NotImplementedError + + +class BuildProcessHelper(BuildProcess): + """A BuildProcess base class with some useful functionality. + """ + @contextmanager + def repo_operation(self): + exc_type = ReleaseVCSError if self.skip_repo_errors else None + try: + yield + except exc_type as e: + print_warning("THE FOLLOWING ERROR WAS SKIPPED:\n%s" % str(e)) + + def visit_variants(self, func, variants=None, **kwargs): + """Iterate over variants and call a function on each.""" + if variants: + present_variants = range(self.package.num_variants) + invalid_variants = set(variants) - set(present_variants) + if invalid_variants: + raise BuildError( + "The package does not contain the variants: %s" + % ", ".join(str(x) for x in sorted(invalid_variants))) + + # iterate over variants + results = [] + num_visited = 0 + + for variant in self.package.iter_variants(): + if variants and variant.index not in variants: + self._print_header( + "Skipping variant %s (%s)..." + % (variant.index, self._n_of_m(variant))) + continue + + # visit the variant + result = func(variant, **kwargs) + results.append(result) + num_visited += 1 + + return num_visited, results + + def get_package_install_path(self, path): + """Return the installation path for a package (where its payload goes). + + Args: + path (str): Package repository path. + """ + from rez.package_repository import package_repository_manager + + pkg_repo = package_repository_manager.get_repository(path) + + return pkg_repo.get_package_payload_path( + package_name=self.package.name, + package_version=self.package.version + ) + + def create_build_context(self, variant, build_type, build_path): + """Create a context to build the variant within.""" + request = variant.get_requires(build_requires=True, + private_build_requires=True) + + req_strs = map(str, request) + quoted_req_strs = map(quote, req_strs) + self._print("Resolving build environment: %s", ' '.join(quoted_req_strs)) + + if build_type == BuildType.local: + packages_path = self.package.config.packages_path + else: + packages_path = self.package.config.nonlocal_packages_path + + # It is uncommon, but possible, to define the package filters in the + # developer package. Example scenario: you may want to enable visiblity + # of *.dev packages if the current package is *.dev also, for example + # (assuming you have a production-time package filter which filters out + # *.dev packages by default). + # + if self.package.config.is_overridden("package_filter"): + from rez.package_filter import PackageFilterList + + data = self.package.config.package_filter + package_filter = PackageFilterList.from_pod(data) + else: + package_filter = None + + # create the build context + context = ResolvedContext(request, + package_paths=packages_path, + package_filter=package_filter, + building=True) + if self.verbose: + context.print_info() + + # save context before possible fail, so user can debug + rxt_filepath = os.path.join(build_path, "build.rxt") + context.save(rxt_filepath) + + if context.status != ResolverStatus.solved: + raise BuildContextResolveError(context) + return context, rxt_filepath + + def pre_release(self): + release_settings = self.package.config.plugins.release_vcs + + # test that the release path exists + release_path = self.package.config.release_packages_path + if not os.path.exists(release_path): + raise ReleaseError("Release path does not exist: %r" % release_path) + + # test that the repo is in a state to release + if self.vcs: + self._print("Checking state of repository...") + with self.repo_operation(): + self.vcs.validate_repostate() + + # check if the repo is already tagged at the current version + if release_settings.check_tag and not self.ignore_existing_tag: + tag_name = self.get_current_tag_name() + tag_exists = False + with self.repo_operation(): + tag_exists = self.vcs.tag_exists(tag_name) + + if tag_exists: + raise ReleaseError( + "Cannot release - the current package version '%s' is " + "already tagged in the repository. Use --ignore-existing-tag " + "to force the release" % self.package.version) + + it = iter_packages(self.package.name, paths=[release_path]) + packages = sorted(it, key=lambda x: x.version, reverse=True) + + # check UUID. This stops unrelated packages that happen to have the same + # name, being released as though they are the same package + if self.package.uuid and packages: + latest_package = packages[0] + if latest_package.uuid and latest_package.uuid != self.package.uuid: + raise ReleaseError( + "Cannot release - the packages are not the same (UUID mismatch)") + + # test that a newer package version hasn't already been released + if self.ensure_latest: + for package in packages: + if package.version > self.package.version: + raise ReleaseError( + "Cannot release - a newer package version already " + "exists (%s)" % package.uri) + else: + break + + def post_release(self, release_message=None): + tag_name = self.get_current_tag_name() + + if self.vcs is None: + return # nothing more to do + + # write a tag for the new release into the vcs + with self.repo_operation(): + self.vcs.create_release_tag(tag_name=tag_name, message=release_message) + + def get_current_tag_name(self): + release_settings = self.package.config.plugins.release_vcs + try: + tag_name = self.package.format(release_settings.tag_name) + except Exception as e: + raise ReleaseError("Error formatting release tag name: %s" % str(e)) + if not tag_name: + tag_name = "unversioned" + return tag_name + + def run_hooks(self, hook_event, **kwargs): + hook_names = self.package.config.release_hooks or [] + hooks = create_release_hooks(hook_names, self.working_dir) + + for hook in hooks: + debug_print("Running %s hook '%s'...", + hook_event.label, hook.name()) + try: + func = getattr(hook, hook_event.__name__) + func(user=getpass.getuser(), **kwargs) + except ReleaseHookCancellingError as e: + raise ReleaseError( + "%s cancelled by %s hook '%s': %s:\n%s" + % (hook_event.noun, hook_event.label, hook.name(), + e.__class__.__name__, str(e))) + except RezError: + debug_print("Error in %s hook '%s': %s:\n%s" + % (hook_event.label, hook.name(), + e.__class__.__name__, str(e))) + + def get_previous_release(self): + release_path = self.package.config.release_packages_path + it = iter_packages(self.package.name, paths=[release_path]) + packages = sorted(it, key=lambda x: x.version, reverse=True) + + for package in packages: + if package.version < self.package.version: + return package + return None + + def get_changelog(self): + previous_package = self.get_previous_release() + if previous_package: + previous_revision = previous_package.revision + else: + previous_revision = None + + changelog = None + with self.repo_operation(): + changelog = self.vcs.get_changelog( + previous_revision, + max_revisions=config.max_package_changelog_revisions) + + return changelog + + def get_release_data(self): + """Get release data for this release. + + Returns: + dict. + """ + previous_package = self.get_previous_release() + if previous_package: + previous_version = previous_package.version + previous_revision = previous_package.revision + else: + previous_version = None + previous_revision = None + + if self.vcs is None: + return dict(vcs="None", + previous_version=previous_version) + + revision = None + with self.repo_operation(): + revision = self.vcs.get_current_revision() + + changelog = self.get_changelog() + + # truncate changelog - very large changelogs can cause package load + # times to be very high, we don't want that + maxlen = config.max_package_changelog_chars + if maxlen and changelog and len(changelog) > maxlen + 3: + changelog = changelog[:maxlen] + "..." + + return dict(vcs=self.vcs.name(), + revision=revision, + changelog=changelog, + previous_version=previous_version, + previous_revision=previous_revision) + + def _print(self, txt, *nargs): + if self.verbose: + if nargs: + txt = txt % nargs + print(txt) + + def _print_header(self, txt, n=1): + if self.quiet: + return + + self._print('') + if n <= 1: + br = '=' * 80 + title = "%s\n%s\n%s" % (br, txt, br) + else: + title = "%s\n%s" % (txt, '-' * len(txt)) + + pr = Printer(sys.stdout) + pr(title, heading) + + def _n_of_m(self, variant): + num_variants = max(self.package.num_variants, 1) + index = (variant.index or 0) + 1 + return "%d/%d" % (index, num_variants) + + +# Copyright 2013-2016 Allan Johns. +# +# This library is free software: you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation, either +# version 3 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . diff --git a/src/rez/build_process_.py b/src/rez/build_process_.py index 80d7b78ac..028e6b9f2 100644 --- a/src/rez/build_process_.py +++ b/src/rez/build_process_.py @@ -1,454 +1,8 @@ -from __future__ import print_function +import warnings +from rez.build_process import * -from rez.packages_ import iter_packages -from rez.exceptions import BuildProcessError, BuildContextResolveError, \ - ReleaseHookCancellingError, RezError, ReleaseError, BuildError, \ - ReleaseVCSError -from rez.utils.logging_ import print_warning -from rez.utils.colorize import heading, Printer -from rez.resolved_context import ResolvedContext -from rez.release_hook import create_release_hooks -from rez.resolver import ResolverStatus -from rez.config import config -from rez.vendor.enum import Enum -from contextlib import contextmanager -from pipes import quote -import getpass -import os.path -import sys - -debug_print = config.debug_printer("package_release") - - -def get_build_process_types(): - """Returns the available build process implementations.""" - from rez.plugin_managers import plugin_manager - return plugin_manager.get_plugins('build_process') - - -def create_build_process(process_type, working_dir, build_system, package=None, - vcs=None, ensure_latest=True, skip_repo_errors=False, - ignore_existing_tag=False, verbose=False, quiet=False): - """Create a `BuildProcess` instance.""" - from rez.plugin_managers import plugin_manager - process_types = get_build_process_types() - if process_type not in process_types: - raise BuildProcessError("Unknown build process: %r" % process_type) - - cls = plugin_manager.get_plugin_class('build_process', process_type) - - return cls(working_dir, # ignored (deprecated) - build_system, - package=package, # ignored (deprecated) - vcs=vcs, - ensure_latest=ensure_latest, - skip_repo_errors=skip_repo_errors, - ignore_existing_tag=ignore_existing_tag, - verbose=verbose, - quiet=quiet) - - -class BuildType(Enum): - """ Enum to represent the type of build.""" - local = 0 - central = 1 - - -class BuildProcess(object): - """A BuildProcess builds and possibly releases a package. - - A build process iterates over the variants of a package, creates the - correct build environment for each variant, builds that variant using a - build system (or possibly creates a script so the user can do that - independently), and then possibly releases the package with the nominated - VCS. This is an abstract base class, you should use a BuildProcess - subclass. - """ - @classmethod - def name(cls): - raise NotImplementedError - - def __init__(self, working_dir, build_system, package=None, vcs=None, - ensure_latest=True, skip_repo_errors=False, - ignore_existing_tag=False, verbose=False, quiet=False): - """Create a BuildProcess. - - Args: - working_dir (DEPRECATED): Ignored. - build_system (`BuildSystem`): Build system used to build the package. - package (DEPRECATED): Ignored. - vcs (`ReleaseVCS`): Version control system to use for the release - process. - ensure_latest: If True, do not allow the release process to occur - if an newer versioned package is already released. - skip_repo_errors: If True, proceed with the release even when errors - occur. BE CAREFUL using this option, it is here in case a package - needs to be released urgently even though there is some problem - with reading or writing the repository. - ignore_existing_tag: Perform the release even if the repository is - already tagged at the current version. If the config setting - plugins.release_vcs.check_tag is False, this has no effect. - verbose (bool): Verbose mode. - quiet (bool): Quiet mode (overrides `verbose`). - """ - self.verbose = verbose and not quiet - self.quiet = quiet - self.build_system = build_system - self.vcs = vcs - self.ensure_latest = ensure_latest - self.skip_repo_errors = skip_repo_errors - self.ignore_existing_tag = ignore_existing_tag - - if vcs and vcs.pkg_root != self.working_dir: - raise BuildProcessError( - "Build process was instantiated with a mismatched VCS instance") - - self.build_path = os.path.join(self.working_dir, - self.package.config.build_directory) - - @property - def package(self): - return self.build_system.package - - @property - def working_dir(self): - return self.build_system.working_dir - - def build(self, install_path=None, clean=False, install=False, variants=None): - """Perform the build process. - - Iterates over the package's variants, resolves the environment for - each, and runs the build system within each resolved environment. - - Args: - install_path (str): The package repository path to install the - package to, if installing. If None, defaults to - `config.local_packages_path`. - clean (bool): If True, clear any previous build first. Otherwise, - rebuild over the top of a previous build. - install (bool): If True, install the build. - variants (list of int): Indexes of variants to build, all if None. - - Raises: - `BuildError`: If the build failed. - - Returns: - int: Number of variants successfully built. - """ - raise NotImplementedError - - def release(self, release_message=None, variants=None): - """Perform the release process. - - Iterates over the package's variants, building and installing each into - the release path determined by `config.release_packages_path`. - - Args: - release_message (str): Message to associate with the release. - variants (list of int): Indexes of variants to release, all if None. - - Raises: - `ReleaseError`: If the release failed. - - Returns: - int: Number of variants successfully released. - """ - raise NotImplementedError - - def get_changelog(self): - """Get the changelog since last package release. - - Returns: - str: Changelog. - """ - raise NotImplementedError - - -class BuildProcessHelper(BuildProcess): - """A BuildProcess base class with some useful functionality. - """ - @contextmanager - def repo_operation(self): - exc_type = ReleaseVCSError if self.skip_repo_errors else None - try: - yield - except exc_type as e: - print_warning("THE FOLLOWING ERROR WAS SKIPPED:\n%s" % str(e)) - - def visit_variants(self, func, variants=None, **kwargs): - """Iterate over variants and call a function on each.""" - if variants: - present_variants = range(self.package.num_variants) - invalid_variants = set(variants) - set(present_variants) - if invalid_variants: - raise BuildError( - "The package does not contain the variants: %s" - % ", ".join(str(x) for x in sorted(invalid_variants))) - - # iterate over variants - results = [] - num_visited = 0 - - for variant in self.package.iter_variants(): - if variants and variant.index not in variants: - self._print_header( - "Skipping variant %s (%s)..." - % (variant.index, self._n_of_m(variant))) - continue - - # visit the variant - result = func(variant, **kwargs) - results.append(result) - num_visited += 1 - - return num_visited, results - - def get_package_install_path(self, path): - """Return the installation path for a package (where its payload goes). - - Args: - path (str): Package repository path. - """ - from rez.package_repository import package_repository_manager - - pkg_repo = package_repository_manager.get_repository(path) - - return pkg_repo.get_package_payload_path( - package_name=self.package.name, - package_version=self.package.version - ) - - def create_build_context(self, variant, build_type, build_path): - """Create a context to build the variant within.""" - request = variant.get_requires(build_requires=True, - private_build_requires=True) - - req_strs = map(str, request) - quoted_req_strs = map(quote, req_strs) - self._print("Resolving build environment: %s", ' '.join(quoted_req_strs)) - - if build_type == BuildType.local: - packages_path = self.package.config.packages_path - else: - packages_path = self.package.config.nonlocal_packages_path - - # It is uncommon, but possible, to define the package filters in the - # developer package. Example scenario: you may want to enable visiblity - # of *.dev packages if the current package is *.dev also, for example - # (assuming you have a production-time package filter which filters out - # *.dev packages by default). - # - if self.package.config.is_overridden("package_filter"): - from rez.package_filter import PackageFilterList - - data = self.package.config.package_filter - package_filter = PackageFilterList.from_pod(data) - else: - package_filter = None - - # create the build context - context = ResolvedContext(request, - package_paths=packages_path, - package_filter=package_filter, - building=True) - if self.verbose: - context.print_info() - - # save context before possible fail, so user can debug - rxt_filepath = os.path.join(build_path, "build.rxt") - context.save(rxt_filepath) - - if context.status != ResolverStatus.solved: - raise BuildContextResolveError(context) - return context, rxt_filepath - - def pre_release(self): - release_settings = self.package.config.plugins.release_vcs - - # test that the release path exists - release_path = self.package.config.release_packages_path - if not os.path.exists(release_path): - raise ReleaseError("Release path does not exist: %r" % release_path) - - # test that the repo is in a state to release - if self.vcs: - self._print("Checking state of repository...") - with self.repo_operation(): - self.vcs.validate_repostate() - - # check if the repo is already tagged at the current version - if release_settings.check_tag and not self.ignore_existing_tag: - tag_name = self.get_current_tag_name() - tag_exists = False - with self.repo_operation(): - tag_exists = self.vcs.tag_exists(tag_name) - - if tag_exists: - raise ReleaseError( - "Cannot release - the current package version '%s' is " - "already tagged in the repository. Use --ignore-existing-tag " - "to force the release" % self.package.version) - - it = iter_packages(self.package.name, paths=[release_path]) - packages = sorted(it, key=lambda x: x.version, reverse=True) - - # check UUID. This stops unrelated packages that happen to have the same - # name, being released as though they are the same package - if self.package.uuid and packages: - latest_package = packages[0] - if latest_package.uuid and latest_package.uuid != self.package.uuid: - raise ReleaseError( - "Cannot release - the packages are not the same (UUID mismatch)") - - # test that a newer package version hasn't already been released - if self.ensure_latest: - for package in packages: - if package.version > self.package.version: - raise ReleaseError( - "Cannot release - a newer package version already " - "exists (%s)" % package.uri) - else: - break - - def post_release(self, release_message=None): - tag_name = self.get_current_tag_name() - - if self.vcs is None: - return # nothing more to do - - # write a tag for the new release into the vcs - with self.repo_operation(): - self.vcs.create_release_tag(tag_name=tag_name, message=release_message) - - def get_current_tag_name(self): - release_settings = self.package.config.plugins.release_vcs - try: - tag_name = self.package.format(release_settings.tag_name) - except Exception as e: - raise ReleaseError("Error formatting release tag name: %s" % str(e)) - if not tag_name: - tag_name = "unversioned" - return tag_name - - def run_hooks(self, hook_event, **kwargs): - hook_names = self.package.config.release_hooks or [] - hooks = create_release_hooks(hook_names, self.working_dir) - - for hook in hooks: - debug_print("Running %s hook '%s'...", - hook_event.label, hook.name()) - try: - func = getattr(hook, hook_event.__name__) - func(user=getpass.getuser(), **kwargs) - except ReleaseHookCancellingError as e: - raise ReleaseError( - "%s cancelled by %s hook '%s': %s:\n%s" - % (hook_event.noun, hook_event.label, hook.name(), - e.__class__.__name__, str(e))) - except RezError: - debug_print("Error in %s hook '%s': %s:\n%s" - % (hook_event.label, hook.name(), - e.__class__.__name__, str(e))) - - def get_previous_release(self): - release_path = self.package.config.release_packages_path - it = iter_packages(self.package.name, paths=[release_path]) - packages = sorted(it, key=lambda x: x.version, reverse=True) - - for package in packages: - if package.version < self.package.version: - return package - return None - - def get_changelog(self): - previous_package = self.get_previous_release() - if previous_package: - previous_revision = previous_package.revision - else: - previous_revision = None - - changelog = None - with self.repo_operation(): - changelog = self.vcs.get_changelog( - previous_revision, - max_revisions=config.max_package_changelog_revisions) - - return changelog - - def get_release_data(self): - """Get release data for this release. - - Returns: - dict. - """ - previous_package = self.get_previous_release() - if previous_package: - previous_version = previous_package.version - previous_revision = previous_package.revision - else: - previous_version = None - previous_revision = None - - if self.vcs is None: - return dict(vcs="None", - previous_version=previous_version) - - revision = None - with self.repo_operation(): - revision = self.vcs.get_current_revision() - - changelog = self.get_changelog() - - # truncate changelog - very large changelogs can cause package load - # times to be very high, we don't want that - maxlen = config.max_package_changelog_chars - if maxlen and changelog and len(changelog) > maxlen + 3: - changelog = changelog[:maxlen] + "..." - - return dict(vcs=self.vcs.name(), - revision=revision, - changelog=changelog, - previous_version=previous_version, - previous_revision=previous_revision) - - def _print(self, txt, *nargs): - if self.verbose: - if nargs: - txt = txt % nargs - print(txt) - - def _print_header(self, txt, n=1): - if self.quiet: - return - - self._print('') - if n <= 1: - br = '=' * 80 - title = "%s\n%s\n%s" % (br, txt, br) - else: - title = "%s\n%s" % (txt, '-' * len(txt)) - - pr = Printer(sys.stdout) - pr(title, heading) - - def _n_of_m(self, variant): - num_variants = max(self.package.num_variants, 1) - index = (variant.index or 0) + 1 - return "%d/%d" % (index, num_variants) - - -# Copyright 2013-2016 Allan Johns. -# -# This library is free software: you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation, either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . +warnings.warn( + "rez.build_process_ is deprecated; import rez.build_process instead", + DeprecationWarning +) diff --git a/src/rez/build_system.py b/src/rez/build_system.py index 4b95708b3..7de7a1512 100644 --- a/src/rez/build_system.py +++ b/src/rez/build_system.py @@ -1,8 +1,8 @@ import os.path -from rez.build_process_ import BuildType +from rez.build_process import BuildType from rez.exceptions import BuildSystemError -from rez.packages_ import get_developer_package +from rez.packages import get_developer_package def get_buildsys_types(): diff --git a/src/rez/cli/_complete_util.py b/src/rez/cli/_complete_util.py index f8570686c..c33637232 100644 --- a/src/rez/cli/_complete_util.py +++ b/src/rez/cli/_complete_util.py @@ -34,12 +34,12 @@ def ConfigCompleter(prefix, **kwargs): def PackageCompleter(prefix, **kwargs): - from rez.packages_ import get_completions + from rez.packages import get_completions return get_completions(prefix) def PackageFamilyCompleter(prefix, **kwargs): - from rez.packages_ import get_completions + from rez.packages import get_completions return get_completions(prefix, family_only=True) diff --git a/src/rez/cli/build.py b/src/rez/cli/build.py index a1a649d34..de4962ecb 100644 --- a/src/rez/cli/build.py +++ b/src/rez/cli/build.py @@ -14,7 +14,7 @@ def get_current_developer_package(): - from rez.packages_ import get_developer_package + from rez.packages import get_developer_package from rez.exceptions import PackageMetadataError global _package @@ -31,7 +31,7 @@ def get_current_developer_package(): def setup_parser_common(parser): """Parser setup common to both rez-build and rez-release.""" - from rez.build_process_ import get_build_process_types + from rez.build_process import get_build_process_types from rez.build_system import get_valid_build_systems process_types = get_build_process_types() @@ -119,7 +119,7 @@ def get_build_args(opts, parser, extra_arg_groups): def command(opts, parser, extra_arg_groups=None): from rez.exceptions import BuildContextResolveError - from rez.build_process_ import create_build_process + from rez.build_process import create_build_process from rez.build_system import create_build_system from rez.serialise import FileFormat import sys diff --git a/src/rez/cli/cp.py b/src/rez/cli/cp.py index d24dcb263..f0d82b82b 100644 --- a/src/rez/cli/cp.py +++ b/src/rez/cli/cp.py @@ -66,7 +66,7 @@ def command(opts, parser, extra_arg_groups=None): from rez.package_repository import package_repository_manager from rez.package_copy import copy_package from rez.utils.formatting import PackageRequest - from rez.packages_ import iter_packages + from rez.packages import iter_packages if (not opts.dest_path) and not (opts.rename or opts.reversion): parser.error("--dest-path must be specified unless --rename or " diff --git a/src/rez/cli/diff.py b/src/rez/cli/diff.py index 2a014547c..622df521e 100644 --- a/src/rez/cli/diff.py +++ b/src/rez/cli/diff.py @@ -19,7 +19,7 @@ def setup_parser(parser, completions=False): def command(opts, parser, extra_arg_groups=None): - from rez.packages_ import get_package_from_string + from rez.packages import get_package_from_string from rez.utils.diff_packages import diff_packages pkg1 = get_package_from_string(opts.PKG1) diff --git a/src/rez/cli/memcache.py b/src/rez/cli/memcache.py index 1e40dcee9..b8ea80976 100644 --- a/src/rez/cli/memcache.py +++ b/src/rez/cli/memcache.py @@ -73,7 +73,7 @@ def poll(client, interval): def command(opts, parser, extra_arg_groups=None): from rez.config import config - from rez.packages_ import iter_package_families, iter_packages + from rez.packages import iter_package_families, iter_packages from rez.utils.yaml import dump_yaml from rez.utils.memcached import Client from rez.utils.formatting import columnise, readable_time_duration, \ diff --git a/src/rez/cli/release.py b/src/rez/cli/release.py index c7bf67efc..f04282a15 100644 --- a/src/rez/cli/release.py +++ b/src/rez/cli/release.py @@ -39,7 +39,7 @@ def setup_parser(parser, completions=False): def command(opts, parser, extra_arg_groups=None): - from rez.build_process_ import create_build_process + from rez.build_process import create_build_process from rez.build_system import create_build_system from rez.release_vcs import create_release_vcs from rez.cli.build import get_build_args, get_current_developer_package diff --git a/src/rez/cli/view.py b/src/rez/cli/view.py index ab2fdbcce..fbfe1c8b0 100644 --- a/src/rez/cli/view.py +++ b/src/rez/cli/view.py @@ -30,7 +30,7 @@ def setup_parser(parser, completions=False): def command(opts, parser, extra_arg_groups=None): from rez.utils.formatting import PackageRequest from rez.serialise import FileFormat - from rez.packages_ import iter_packages + from rez.packages import iter_packages from rez.status import status import sys diff --git a/src/rez/cli/yaml2py.py b/src/rez/cli/yaml2py.py index a52f6cd3d..79154c259 100644 --- a/src/rez/cli/yaml2py.py +++ b/src/rez/cli/yaml2py.py @@ -12,7 +12,7 @@ def setup_parser(parser, completions=False): def command(opts, parser, extra_arg_groups=None): - from rez.packages_ import get_developer_package + from rez.packages import get_developer_package from rez.serialise import FileFormat from rez.exceptions import PackageMetadataError import os.path diff --git a/src/rez/developer_package.py b/src/rez/developer_package.py index acb6f08a3..10badc0cc 100644 --- a/src/rez/developer_package.py +++ b/src/rez/developer_package.py @@ -1,7 +1,6 @@ from rez.config import config -from rez.packages_ import Package +from rez.packages import Package, create_package from rez.serialise import load_from_file, FileFormat, set_objects -from rez.packages_ import create_package from rez.exceptions import PackageMetadataError, InvalidPackageError from rez.utils.execution import add_sys_paths from rez.utils.sourcecode import SourceCode diff --git a/src/rez/package_filter.py b/src/rez/package_filter.py index 7092bf19f..961b3dd74 100644 --- a/src/rez/package_filter.py +++ b/src/rez/package_filter.py @@ -1,4 +1,4 @@ -from rez.packages_ import iter_packages +from rez.packages import iter_packages from rez.exceptions import ConfigurationError from rez.config import config from rez.utils.data_utils import cached_property, cached_class_property diff --git a/src/rez/package_help.py b/src/rez/package_help.py index 2a02682af..0723fb515 100644 --- a/src/rez/package_help.py +++ b/src/rez/package_help.py @@ -1,6 +1,6 @@ from __future__ import print_function -from rez.packages_ import iter_packages +from rez.packages import iter_packages from rez.config import config from rez.rex_bindings import VersionBinding from rez.utils.execution import Popen diff --git a/src/rez/package_maker.py b/src/rez/package_maker.py new file mode 100644 index 000000000..4a77d1b8d --- /dev/null +++ b/src/rez/package_maker.py @@ -0,0 +1,249 @@ +from rez.utils._version import _rez_version +from rez.utils.schema import Required, schema_keys, extensible_schema_dict +from rez.utils.filesystem import retain_cwd +from rez.utils.formatting import PackageRequest +from rez.utils.data_utils import AttrDictWrapper +from rez.utils.logging_ import print_warning +from rez.exceptions import PackageMetadataError +from rez.package_resources import help_schema, _commands_schema, \ + _function_schema, late_bound +from rez.package_repository import create_memory_package_repository +from rez.packages import Package +from rez.package_py_utils import expand_requirement +from rez.vendor.schema.schema import Schema, Optional, Or, Use, And +from rez.vendor.six import six +from rez.vendor.version.version import Version +from contextlib import contextmanager +import os + + +basestring = six.string_types[0] + + +# this schema will automatically harden request strings like 'python-*'; see +# the 'expand_requires' function for more info. +# +package_request_schema = Or(And(basestring, Use(expand_requirement)), + And(PackageRequest, Use(str))) + +tests_schema = Schema({ + Optional(basestring): Or( + Or(basestring, [basestring]), + extensible_schema_dict({ + "command": Or(basestring, [basestring]), + Optional("requires"): [package_request_schema], + Optional("run_on"): Or(basestring, [basestring]), + Optional("on_variants"): Or( + bool, + { + "type": "requires", + "value": [package_request_schema] + } + ) + }) + ) +}) + + +package_schema = Schema({ + Optional("requires_rez_version"): And(basestring, Use(Version)), + + Required("name"): basestring, + Optional("base"): basestring, + Optional("version"): Or(basestring, + And(Version, Use(str))), + Optional('description'): basestring, + Optional('authors'): [basestring], + + Optional('requires'): late_bound([package_request_schema]), + Optional('build_requires'): late_bound([package_request_schema]), + Optional('private_build_requires'): late_bound([package_request_schema]), + + # deliberately not possible to late bind + Optional('variants'): [[package_request_schema]], + + Optional('relocatable'): late_bound(Or(None, bool)), + Optional('hashed_variants'): bool, + + Optional('uuid'): basestring, + Optional('config'): dict, + Optional('tools'): late_bound([basestring]), + Optional('help'): late_bound(help_schema), + + Optional('tests'): late_bound(tests_schema), + + Optional('pre_commands'): _commands_schema, + Optional('commands'): _commands_schema, + Optional('post_commands'): _commands_schema, + Optional('pre_build_commands'): _commands_schema, + + # attributes specific to pre-built packages + Optional("build_system"): basestring, + Optional("build_command"): Or([basestring], basestring, False), + Optional("preprocess"): _function_schema, + + # arbitrary fields + Optional(basestring): object +}) + + +class PackageMaker(AttrDictWrapper): + """Utility class for creating packages.""" + def __init__(self, name, data=None, package_cls=None): + """Create a package maker. + + Args: + name (str): Package name. + """ + super(PackageMaker, self).__init__(data) + self.name = name + self.package_cls = package_cls or Package + + # set by `make_package` + self.installed_variants = [] + self.skipped_variants = [] + + def get_package(self): + """Create the analogous package. + + Returns: + `Package` object. + """ + # get and validate package data + package_data = self._get_data() + package_data = package_schema.validate(package_data) + + # check compatibility with rez version + if "requires_rez_version" in package_data: + ver = package_data.pop("requires_rez_version") + + if Version(_rez_version) < ver: + raise PackageMetadataError( + "Failed reading package definition file: rez version >= %s " + "needed (current version is %s)" % (ver, _rez_version) + ) + + # create a 'memory' package repository containing just this package + version_str = package_data.get("version") or "_NO_VERSION" + repo_data = {self.name: {version_str: package_data}} + repo = create_memory_package_repository(repo_data) + + # retrieve the package from the new repository + family_resource = repo.get_package_family(self.name) + it = repo.iter_packages(family_resource) + package_resource = next(it) + + package = self.package_cls(package_resource) + + # revalidate the package for extra measure + package.validate_data() + return package + + def _get_data(self): + data = self._data.copy() + + data.pop("installed_variants", None) + data.pop("skipped_variants", None) + data.pop("package_cls", None) + + data = dict((k, v) for k, v in data.items() if v is not None) + return data + + +@contextmanager +def make_package(name, path, make_base=None, make_root=None, skip_existing=True, + warn_on_skip=True): + """Make and install a package. + + Example: + + >>> def make_root(variant, path): + >>> os.symlink("/foo_payload/misc/python27", "ext") + >>> + >>> with make_package('foo', '/packages', make_root=make_root) as pkg: + >>> pkg.version = '1.0.0' + >>> pkg.description = 'does foo things' + >>> pkg.requires = ['python-2.7'] + + Args: + name (str): Package name. + path (str): Package repository path to install package into. + make_base (callable): Function that is used to create the package + payload, if applicable. + make_root (callable): Function that is used to create the package + variant payloads, if applicable. + skip_existing (bool): If True, detect if a variant already exists, and + skip with a warning message if so. + warn_on_skip (bool): If True, print warning when a variant is skipped. + + Yields: + `PackageMaker` object. + + Note: + Both `make_base` and `make_root` are called once per variant install, + and have the signature (variant, path). + + Note: + The 'installed_variants' attribute on the `PackageMaker` instance will + be appended with variant(s) created by this function, if any. + """ + maker = PackageMaker(name) + yield maker + + # post-with-block: + # + + package = maker.get_package() + cwd = os.getcwd() + src_variants = [] + + # skip those variants that already exist + if skip_existing: + for variant in package.iter_variants(): + variant_ = variant.install(path, dry_run=True) + if variant_ is None: + src_variants.append(variant) + else: + maker.skipped_variants.append(variant_) + if warn_on_skip: + print_warning("Skipping installation: Package variant already " + "exists: %s" % variant_.uri) + else: + src_variants = package.iter_variants() + + with retain_cwd(): + # install the package variant(s) into the filesystem package repo at `path` + for variant in src_variants: + variant_ = variant.install(path) + + base = variant_.base + if make_base and base: + if not os.path.exists(base): + os.makedirs(base) + os.chdir(base) + make_base(variant_, base) + + root = variant_.root + if make_root and root: + if not os.path.exists(root): + os.makedirs(root) + os.chdir(root) + make_root(variant_, root) + + maker.installed_variants.append(variant_) + + +# Copyright 2013-2016 Allan Johns. +# +# This library is free software: you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation, either +# version 3 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . diff --git a/src/rez/package_maker__.py b/src/rez/package_maker__.py index c3b9c4cbb..e6de7747e 100644 --- a/src/rez/package_maker__.py +++ b/src/rez/package_maker__.py @@ -1,249 +1,8 @@ -from rez.utils._version import _rez_version -from rez.utils.schema import Required, schema_keys, extensible_schema_dict -from rez.utils.filesystem import retain_cwd -from rez.utils.formatting import PackageRequest -from rez.utils.data_utils import AttrDictWrapper -from rez.utils.logging_ import print_warning -from rez.exceptions import PackageMetadataError -from rez.package_resources_ import help_schema, _commands_schema, \ - _function_schema, late_bound -from rez.package_repository import create_memory_package_repository -from rez.packages_ import Package -from rez.package_py_utils import expand_requirement -from rez.vendor.schema.schema import Schema, Optional, Or, Use, And -from rez.vendor.six import six -from rez.vendor.version.version import Version -from contextlib import contextmanager -import os +import warnings +from rez.package_maker import * -basestring = six.string_types[0] - - -# this schema will automatically harden request strings like 'python-*'; see -# the 'expand_requires' function for more info. -# -package_request_schema = Or(And(basestring, Use(expand_requirement)), - And(PackageRequest, Use(str))) - -tests_schema = Schema({ - Optional(basestring): Or( - Or(basestring, [basestring]), - extensible_schema_dict({ - "command": Or(basestring, [basestring]), - Optional("requires"): [package_request_schema], - Optional("run_on"): Or(basestring, [basestring]), - Optional("on_variants"): Or( - bool, - { - "type": "requires", - "value": [package_request_schema] - } - ) - }) - ) -}) - - -package_schema = Schema({ - Optional("requires_rez_version"): And(basestring, Use(Version)), - - Required("name"): basestring, - Optional("base"): basestring, - Optional("version"): Or(basestring, - And(Version, Use(str))), - Optional('description'): basestring, - Optional('authors'): [basestring], - - Optional('requires'): late_bound([package_request_schema]), - Optional('build_requires'): late_bound([package_request_schema]), - Optional('private_build_requires'): late_bound([package_request_schema]), - - # deliberately not possible to late bind - Optional('variants'): [[package_request_schema]], - - Optional('relocatable'): late_bound(Or(None, bool)), - Optional('hashed_variants'): bool, - - Optional('uuid'): basestring, - Optional('config'): dict, - Optional('tools'): late_bound([basestring]), - Optional('help'): late_bound(help_schema), - - Optional('tests'): late_bound(tests_schema), - - Optional('pre_commands'): _commands_schema, - Optional('commands'): _commands_schema, - Optional('post_commands'): _commands_schema, - Optional('pre_build_commands'): _commands_schema, - - # attributes specific to pre-built packages - Optional("build_system"): basestring, - Optional("build_command"): Or([basestring], basestring, False), - Optional("preprocess"): _function_schema, - - # arbitrary fields - Optional(basestring): object -}) - - -class PackageMaker(AttrDictWrapper): - """Utility class for creating packages.""" - def __init__(self, name, data=None, package_cls=None): - """Create a package maker. - - Args: - name (str): Package name. - """ - super(PackageMaker, self).__init__(data) - self.name = name - self.package_cls = package_cls or Package - - # set by `make_package` - self.installed_variants = [] - self.skipped_variants = [] - - def get_package(self): - """Create the analogous package. - - Returns: - `Package` object. - """ - # get and validate package data - package_data = self._get_data() - package_data = package_schema.validate(package_data) - - # check compatibility with rez version - if "requires_rez_version" in package_data: - ver = package_data.pop("requires_rez_version") - - if Version(_rez_version) < ver: - raise PackageMetadataError( - "Failed reading package definition file: rez version >= %s " - "needed (current version is %s)" % (ver, _rez_version) - ) - - # create a 'memory' package repository containing just this package - version_str = package_data.get("version") or "_NO_VERSION" - repo_data = {self.name: {version_str: package_data}} - repo = create_memory_package_repository(repo_data) - - # retrieve the package from the new repository - family_resource = repo.get_package_family(self.name) - it = repo.iter_packages(family_resource) - package_resource = next(it) - - package = self.package_cls(package_resource) - - # revalidate the package for extra measure - package.validate_data() - return package - - def _get_data(self): - data = self._data.copy() - - data.pop("installed_variants", None) - data.pop("skipped_variants", None) - data.pop("package_cls", None) - - data = dict((k, v) for k, v in data.items() if v is not None) - return data - - -@contextmanager -def make_package(name, path, make_base=None, make_root=None, skip_existing=True, - warn_on_skip=True): - """Make and install a package. - - Example: - - >>> def make_root(variant, path): - >>> os.symlink("/foo_payload/misc/python27", "ext") - >>> - >>> with make_package('foo', '/packages', make_root=make_root) as pkg: - >>> pkg.version = '1.0.0' - >>> pkg.description = 'does foo things' - >>> pkg.requires = ['python-2.7'] - - Args: - name (str): Package name. - path (str): Package repository path to install package into. - make_base (callable): Function that is used to create the package - payload, if applicable. - make_root (callable): Function that is used to create the package - variant payloads, if applicable. - skip_existing (bool): If True, detect if a variant already exists, and - skip with a warning message if so. - warn_on_skip (bool): If True, print warning when a variant is skipped. - - Yields: - `PackageMaker` object. - - Note: - Both `make_base` and `make_root` are called once per variant install, - and have the signature (variant, path). - - Note: - The 'installed_variants' attribute on the `PackageMaker` instance will - be appended with variant(s) created by this function, if any. - """ - maker = PackageMaker(name) - yield maker - - # post-with-block: - # - - package = maker.get_package() - cwd = os.getcwd() - src_variants = [] - - # skip those variants that already exist - if skip_existing: - for variant in package.iter_variants(): - variant_ = variant.install(path, dry_run=True) - if variant_ is None: - src_variants.append(variant) - else: - maker.skipped_variants.append(variant_) - if warn_on_skip: - print_warning("Skipping installation: Package variant already " - "exists: %s" % variant_.uri) - else: - src_variants = package.iter_variants() - - with retain_cwd(): - # install the package variant(s) into the filesystem package repo at `path` - for variant in src_variants: - variant_ = variant.install(path) - - base = variant_.base - if make_base and base: - if not os.path.exists(base): - os.makedirs(base) - os.chdir(base) - make_base(variant_, base) - - root = variant_.root - if make_root and root: - if not os.path.exists(root): - os.makedirs(root) - os.chdir(root) - make_root(variant_, root) - - maker.installed_variants.append(variant_) - - -# Copyright 2013-2016 Allan Johns. -# -# This library is free software: you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation, either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . +warnings.warn( + "rez.package_maker__ is deprecated; import rez.package_maker instead", + DeprecationWarning +) diff --git a/src/rez/package_py_utils.py b/src/rez/package_py_utils.py index 43130608d..4b25628e9 100644 --- a/src/rez/package_py_utils.py +++ b/src/rez/package_py_utils.py @@ -54,7 +54,7 @@ def expand_requirement(request, paths=None): from rez.vendor.version.version import VersionRange from rez.vendor.version.requirement import Requirement - from rez.packages_ import get_latest_package + from rez.packages import get_latest_package from uuid import uuid4 wildcard_map = {} @@ -229,7 +229,7 @@ def find_site_python(module_name, paths=None): Returns: `Package`: Native python package containing the named module. """ - from rez.packages_ import iter_packages + from rez.packages import iter_packages import subprocess import ast import os diff --git a/src/rez/package_resources.py b/src/rez/package_resources.py new file mode 100644 index 000000000..6b1bc4a91 --- /dev/null +++ b/src/rez/package_resources.py @@ -0,0 +1,528 @@ +from rez.utils.resources import Resource +from rez.utils.schema import Required, schema_keys, extensible_schema_dict +from rez.utils.logging_ import print_warning +from rez.utils.sourcecode import SourceCode +from rez.utils.data_utils import cached_property, AttributeForwardMeta, \ + LazyAttributeMeta +from rez.utils.filesystem import find_matching_symlink +from rez.utils.formatting import PackageRequest +from rez.exceptions import PackageMetadataError, ResourceError +from rez.config import config, Config, create_config +from rez.vendor.version.version import Version +from rez.vendor.schema.schema import Schema, SchemaError, Optional, Or, And, Use +from rez.vendor.six import six + +from textwrap import dedent +import os.path +from hashlib import sha1 + + +basestring = six.string_types[0] + + +# package attributes created at release time +package_release_keys = ( + "timestamp", + 'revision', + 'changelog', + 'release_message', + 'previous_version', + 'previous_revision', + 'vcs') + +# package attributes that we don't install +package_build_only_keys = ( + "requires_rez_version", + "build_system", + "build_command", + "preprocess", + "pre_build_commands" +) + +# package attributes that are rex-based functions +package_rex_keys = ( + "pre_commands", + "commands", + "post_commands", + "pre_build_commands" +) + + +# ------------------------------------------------------------------------------ +# utility schemas +# ------------------------------------------------------------------------------ + +help_schema = Or(basestring, # single help entry + [[basestring]]) # multiple help entries + +_is_late = And(SourceCode, lambda x: hasattr(x, "_late")) + +def late_bound(schema): + return Or(SourceCode, schema) + +# used when 'requires' is late bound +late_requires_schema = Schema([ + Or(PackageRequest, And(basestring, Use(PackageRequest))) +]) + + +# ------------------------------------------------------------------------------ +# schema dicts +# ------------------------------------------------------------------------------ + +# requirements of all package-related resources +# + +base_resource_schema_dict = { + Required("name"): basestring +} + + +# package family +# + +package_family_schema_dict = base_resource_schema_dict.copy() + + +# schema common to both package and variant +# + +tests_schema = Schema({ + Optional(basestring): Or( + Or(basestring, [basestring]), + extensible_schema_dict({ + "command": Or(basestring, [basestring]), + Optional("requires"): [ + Or(PackageRequest, And(basestring, Use(PackageRequest))) + ], + Optional("run_on"): Or(basestring, [basestring]), + Optional("on_variants"): Or( + bool, + { + "type": "requires", + "value": [ + Or(PackageRequest, And(basestring, Use(PackageRequest))) + ] + } + ) + }) + ) +}) + +package_base_schema_dict = base_resource_schema_dict.copy() +package_base_schema_dict.update({ + # basics + Optional("base"): basestring, + Optional("version"): Version, + Optional('description'): basestring, + Optional('authors'): [basestring], + + # dependencies + Optional('requires'): late_bound([PackageRequest]), + Optional('build_requires'): late_bound([PackageRequest]), + Optional('private_build_requires'): late_bound([PackageRequest]), + + # plugins + Optional('has_plugins'): late_bound(bool), + Optional('plugin_for'): late_bound([basestring]), + + # general + Optional('uuid'): basestring, + Optional('config'): Config, + Optional('tools'): late_bound([basestring]), + Optional('help'): late_bound(help_schema), + + # build related + Optional('relocatable'): late_bound(Or(None, bool)), + Optional('hashed_variants'): bool, + + # testing + Optional('tests'): late_bound(tests_schema), + + # commands + Optional('pre_commands'): SourceCode, + Optional('commands'): SourceCode, + Optional('post_commands'): SourceCode, + Optional('pre_build_commands'): SourceCode, + + # release info + Optional("timestamp"): int, + Optional('revision'): object, + Optional('changelog'): basestring, + Optional('release_message'): Or(None, basestring), + Optional('previous_version'): Version, + Optional('previous_revision'): object, + Optional('vcs'): basestring, + + # arbitrary fields + Optional(basestring): late_bound(object) +}) + + +# package +package_schema_dict = package_base_schema_dict.copy() +package_schema_dict.update({ + # deliberately not possible to late bind + Optional("variants"): [[PackageRequest]] +}) + + +# variant +variant_schema_dict = package_base_schema_dict.copy() + + +# ------------------------------------------------------------------------------ +# resource schemas +# ------------------------------------------------------------------------------ + +package_family_schema = Schema(package_family_schema_dict) + + +package_schema = Schema(package_schema_dict) + + +variant_schema = Schema(variant_schema_dict) + + +# ------------------------------------------------------------------------------ +# schemas for converting from POD datatypes +# ------------------------------------------------------------------------------ + +_commands_schema = Or(SourceCode, # commands as converted function + callable, # commands as function + basestring, # commands in text block + [basestring]) # old-style (rez-1) commands + +_function_schema = Or(SourceCode, callable) + +_package_request_schema = And(basestring, Use(PackageRequest)) + +package_pod_schema_dict = base_resource_schema_dict.copy() + +large_string_dict = And(basestring, Use(lambda x: dedent(x).strip())) + + +package_pod_schema_dict.update({ + Optional("base"): basestring, + Optional("version"): And(basestring, Use(Version)), + Optional('description'): large_string_dict, + Optional('authors'): [basestring], + + Optional('requires'): late_bound([_package_request_schema]), + Optional('build_requires'): late_bound([_package_request_schema]), + Optional('private_build_requires'): late_bound([_package_request_schema]), + + # deliberately not possible to late bind + Optional('variants'): [[_package_request_schema]], + + Optional('has_plugins'): late_bound(bool), + Optional('plugin_for'): late_bound([basestring]), + + Optional('uuid'): basestring, + Optional('config'): And(dict, + Use(lambda x: create_config(overrides=x))), + Optional('tools'): late_bound([basestring]), + Optional('help'): late_bound(help_schema), + + Optional('relocatable'): late_bound(Or(None, bool)), + Optional('hashed_variants'): bool, + + Optional('tests'): late_bound(tests_schema), + + Optional('pre_commands'): _commands_schema, + Optional('commands'): _commands_schema, + Optional('post_commands'): _commands_schema, + Optional('pre_build_commands'): _commands_schema, + + Optional("timestamp"): int, + Optional('revision'): object, + Optional('changelog'): large_string_dict, + Optional('release_message'): Or(None, basestring), + Optional('previous_version'): And(basestring, Use(Version)), + Optional('previous_revision'): object, + Optional('vcs'): basestring, + + # arbitrary keys + Optional(basestring): late_bound(object) +}) + + +package_pod_schema = Schema(package_pod_schema_dict) + + +# ------------------------------------------------------------------------------ +# resource classes +# ------------------------------------------------------------------------------ + +class PackageRepositoryResource(Resource): + """Base class for all package-related resources. + + Attributes: + schema_error (`Exception`): Type of exception to throw on bad data. + repository_type (str): Type of package repository associated with this + resource type. + """ + schema_error = PackageMetadataError + repository_type = None + + @classmethod + def normalize_variables(cls, variables): + if "repository_type" not in variables or "location" not in \ + variables: + raise ResourceError("%s resources require a repository_type and " + "location" % cls.__name__) + return super(PackageRepositoryResource, cls).normalize_variables( + variables) + + def __init__(self, variables=None): + super(PackageRepositoryResource, self).__init__(variables) + + @cached_property + def uri(self): + return self._uri() + + @property + def location(self): + return self.get("location") + + @property + def name(self): + return self.get("name") + + def _uri(self): + """Return a URI. + + Implement this function to return a short, readable string that + uniquely identifies this resource. + """ + raise NotImplementedError + + +class PackageFamilyResource(PackageRepositoryResource): + """A package family. + + A repository implementation's package family resource(s) must derive from + this class. It must satisfy the schema `package_family_schema`. + """ + pass + + +class PackageResource(PackageRepositoryResource): + """A package. + + A repository implementation's package resource(s) must derive from this + class. It must satisfy the schema `package_schema`. + """ + + @classmethod + def normalize_variables(cls, variables): + """Make sure version is treated consistently + """ + # if the version is False, empty string, etc, throw it out + if variables.get('version', True) in ('', False, '_NO_VERSION', None): + del variables['version'] + return super(PackageResource, cls).normalize_variables(variables) + + @cached_property + def version(self): + ver_str = self.get("version", "") + return Version(ver_str) + + +class VariantResource(PackageResource): + """A package variant. + + A repository implementation's variant resource(s) must derive from this + class. It must satisfy the schema `variant_schema`. + + Even packages that do not have a 'variants' section contain a variant - in + this case it is the 'None' variant (the value of `index` is None). This + provides some internal consistency and simplifies the implementation. + """ + @property + def index(self): + return self.get("index", None) + + @cached_property + def root(self): + """Return the 'root' path of the variant.""" + return self._root() + + @cached_property + def subpath(self): + """Return the variant's 'subpath' + + The subpath is the relative path the variant's payload should be stored + under, relative to the package base. If None, implies that the variant + root matches the package base. + """ + return self._subpath() + + def _root(self, ignore_shortlinks=False): + raise NotImplementedError + + def _subpath(self, ignore_shortlinks=False): + raise NotImplementedError + + +# ------------------------------------------------------------------------------ +# resource helper classes +# +# Package repository plugins are not required to use the following classes, but +# they may help minimise the amount of code you need to write. +# ------------------------------------------------------------------------------ + +class PackageResourceHelper(PackageResource): + """PackageResource with some common functionality included. + """ + variant_key = None + + @cached_property + def commands(self): + return self._convert_to_rex(self._commands) + + @cached_property + def pre_commands(self): + return self._convert_to_rex(self._pre_commands) + + @cached_property + def post_commands(self): + return self._convert_to_rex(self._post_commands) + + def iter_variants(self): + num_variants = len(self.variants or []) + + if num_variants == 0: + indexes = [None] + else: + indexes = range(num_variants) + + for index in indexes: + variant = self._repository.get_resource( + self.variant_key, + location=self.location, + name=self.name, + version=self.get("version"), + index=index) + yield variant + + def _convert_to_rex(self, commands): + if isinstance(commands, list): + from rez.utils.backcompat import convert_old_commands + + msg = "package %r is using old-style commands." % self.uri + if config.disable_rez_1_compatibility or config.error_old_commands: + raise SchemaError(None, msg) + elif config.warn("old_commands"): + print_warning(msg) + commands = convert_old_commands(commands) + + if isinstance(commands, basestring): + return SourceCode(source=commands) + elif callable(commands): + return SourceCode(func=commands) + else: + return commands + +class _Metas(AttributeForwardMeta, LazyAttributeMeta): + pass + +class VariantResourceHelper(six.with_metaclass(_Metas, VariantResource)): + """Helper class for implementing variants that inherit properties from their + parent package. + + Since a variant overlaps so much with a package, here we use the forwarding + metaclass to forward our parent package's attributes onto ourself (with some + exceptions - eg 'variants', 'requires'). This is a common enough pattern + that it's supplied here for other repository plugins to use. + """ + + # Note: lazy key validation doesn't happen in this class, it just fowards on + # attributes from the package. But LazyAttributeMeta does still use this + # schema to create other class attributes, such as `validate_data`. + schema = variant_schema + + # forward Package attributes onto ourself + keys = schema_keys(package_schema) - set(["variants"]) + + def _uri(self): + index = self.index + idxstr = '' if index is None else str(index) + return "%s[%s]" % (self.parent.uri, idxstr) + + def _subpath(self, ignore_shortlinks=False): + if self.index is None: + return None + + if self.parent.hashed_variants: + vars_str = str(list(map(str, self.variant_requires))) + h = sha1(vars_str.encode("utf8")) + hashdir = h.hexdigest() + + if (not ignore_shortlinks) and \ + config.use_variant_shortlinks and \ + self.base is not None: + + # search for matching shortlink and use that + path = os.path.join(self.base, config.variant_shortlinks_dirname) + + if os.path.exists(path): + actual_root = os.path.join(self.base, hashdir) + linkname = find_matching_symlink(path, actual_root) + + if linkname: + return os.path.join( + config.variant_shortlinks_dirname, linkname) + + return hashdir + else: + dirs = [x.safe_str() for x in self.variant_requires] + subpath = os.path.join(*dirs) + return subpath + + def _root(self, ignore_shortlinks=False): + if self.base is None: + return None + elif self.index is None: + return self.base + else: + subpath = self._subpath(ignore_shortlinks=ignore_shortlinks) + root = os.path.join(self.base, subpath) + return root + + @cached_property + def variant_requires(self): + index = self.index + if index is None: + return [] + else: + try: + return self.parent.variants[index] or [] + except (IndexError, TypeError): + raise ResourceError( + "Unexpected error - variant %s cannot be found in its " + "parent package %s" % (self.uri, self.parent.uri)) + + @property + def wrapped(self): # forward Package attributes onto ourself + return self.parent + + def _load(self): + # doesn't have its own data, forwards on from parent instead + return None + + +# Copyright 2013-2016 Allan Johns. +# +# This library is free software: you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation, either +# version 3 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . diff --git a/src/rez/package_resources_.py b/src/rez/package_resources_.py index 6b1bc4a91..773f7da8a 100644 --- a/src/rez/package_resources_.py +++ b/src/rez/package_resources_.py @@ -1,528 +1,8 @@ -from rez.utils.resources import Resource -from rez.utils.schema import Required, schema_keys, extensible_schema_dict -from rez.utils.logging_ import print_warning -from rez.utils.sourcecode import SourceCode -from rez.utils.data_utils import cached_property, AttributeForwardMeta, \ - LazyAttributeMeta -from rez.utils.filesystem import find_matching_symlink -from rez.utils.formatting import PackageRequest -from rez.exceptions import PackageMetadataError, ResourceError -from rez.config import config, Config, create_config -from rez.vendor.version.version import Version -from rez.vendor.schema.schema import Schema, SchemaError, Optional, Or, And, Use -from rez.vendor.six import six +import warnings +from rez.package_resources import * -from textwrap import dedent -import os.path -from hashlib import sha1 - -basestring = six.string_types[0] - - -# package attributes created at release time -package_release_keys = ( - "timestamp", - 'revision', - 'changelog', - 'release_message', - 'previous_version', - 'previous_revision', - 'vcs') - -# package attributes that we don't install -package_build_only_keys = ( - "requires_rez_version", - "build_system", - "build_command", - "preprocess", - "pre_build_commands" +warnings.warn( + "rez.package_resources_ is deprecated; import rez.package_resources instead", + DeprecationWarning ) - -# package attributes that are rex-based functions -package_rex_keys = ( - "pre_commands", - "commands", - "post_commands", - "pre_build_commands" -) - - -# ------------------------------------------------------------------------------ -# utility schemas -# ------------------------------------------------------------------------------ - -help_schema = Or(basestring, # single help entry - [[basestring]]) # multiple help entries - -_is_late = And(SourceCode, lambda x: hasattr(x, "_late")) - -def late_bound(schema): - return Or(SourceCode, schema) - -# used when 'requires' is late bound -late_requires_schema = Schema([ - Or(PackageRequest, And(basestring, Use(PackageRequest))) -]) - - -# ------------------------------------------------------------------------------ -# schema dicts -# ------------------------------------------------------------------------------ - -# requirements of all package-related resources -# - -base_resource_schema_dict = { - Required("name"): basestring -} - - -# package family -# - -package_family_schema_dict = base_resource_schema_dict.copy() - - -# schema common to both package and variant -# - -tests_schema = Schema({ - Optional(basestring): Or( - Or(basestring, [basestring]), - extensible_schema_dict({ - "command": Or(basestring, [basestring]), - Optional("requires"): [ - Or(PackageRequest, And(basestring, Use(PackageRequest))) - ], - Optional("run_on"): Or(basestring, [basestring]), - Optional("on_variants"): Or( - bool, - { - "type": "requires", - "value": [ - Or(PackageRequest, And(basestring, Use(PackageRequest))) - ] - } - ) - }) - ) -}) - -package_base_schema_dict = base_resource_schema_dict.copy() -package_base_schema_dict.update({ - # basics - Optional("base"): basestring, - Optional("version"): Version, - Optional('description'): basestring, - Optional('authors'): [basestring], - - # dependencies - Optional('requires'): late_bound([PackageRequest]), - Optional('build_requires'): late_bound([PackageRequest]), - Optional('private_build_requires'): late_bound([PackageRequest]), - - # plugins - Optional('has_plugins'): late_bound(bool), - Optional('plugin_for'): late_bound([basestring]), - - # general - Optional('uuid'): basestring, - Optional('config'): Config, - Optional('tools'): late_bound([basestring]), - Optional('help'): late_bound(help_schema), - - # build related - Optional('relocatable'): late_bound(Or(None, bool)), - Optional('hashed_variants'): bool, - - # testing - Optional('tests'): late_bound(tests_schema), - - # commands - Optional('pre_commands'): SourceCode, - Optional('commands'): SourceCode, - Optional('post_commands'): SourceCode, - Optional('pre_build_commands'): SourceCode, - - # release info - Optional("timestamp"): int, - Optional('revision'): object, - Optional('changelog'): basestring, - Optional('release_message'): Or(None, basestring), - Optional('previous_version'): Version, - Optional('previous_revision'): object, - Optional('vcs'): basestring, - - # arbitrary fields - Optional(basestring): late_bound(object) -}) - - -# package -package_schema_dict = package_base_schema_dict.copy() -package_schema_dict.update({ - # deliberately not possible to late bind - Optional("variants"): [[PackageRequest]] -}) - - -# variant -variant_schema_dict = package_base_schema_dict.copy() - - -# ------------------------------------------------------------------------------ -# resource schemas -# ------------------------------------------------------------------------------ - -package_family_schema = Schema(package_family_schema_dict) - - -package_schema = Schema(package_schema_dict) - - -variant_schema = Schema(variant_schema_dict) - - -# ------------------------------------------------------------------------------ -# schemas for converting from POD datatypes -# ------------------------------------------------------------------------------ - -_commands_schema = Or(SourceCode, # commands as converted function - callable, # commands as function - basestring, # commands in text block - [basestring]) # old-style (rez-1) commands - -_function_schema = Or(SourceCode, callable) - -_package_request_schema = And(basestring, Use(PackageRequest)) - -package_pod_schema_dict = base_resource_schema_dict.copy() - -large_string_dict = And(basestring, Use(lambda x: dedent(x).strip())) - - -package_pod_schema_dict.update({ - Optional("base"): basestring, - Optional("version"): And(basestring, Use(Version)), - Optional('description'): large_string_dict, - Optional('authors'): [basestring], - - Optional('requires'): late_bound([_package_request_schema]), - Optional('build_requires'): late_bound([_package_request_schema]), - Optional('private_build_requires'): late_bound([_package_request_schema]), - - # deliberately not possible to late bind - Optional('variants'): [[_package_request_schema]], - - Optional('has_plugins'): late_bound(bool), - Optional('plugin_for'): late_bound([basestring]), - - Optional('uuid'): basestring, - Optional('config'): And(dict, - Use(lambda x: create_config(overrides=x))), - Optional('tools'): late_bound([basestring]), - Optional('help'): late_bound(help_schema), - - Optional('relocatable'): late_bound(Or(None, bool)), - Optional('hashed_variants'): bool, - - Optional('tests'): late_bound(tests_schema), - - Optional('pre_commands'): _commands_schema, - Optional('commands'): _commands_schema, - Optional('post_commands'): _commands_schema, - Optional('pre_build_commands'): _commands_schema, - - Optional("timestamp"): int, - Optional('revision'): object, - Optional('changelog'): large_string_dict, - Optional('release_message'): Or(None, basestring), - Optional('previous_version'): And(basestring, Use(Version)), - Optional('previous_revision'): object, - Optional('vcs'): basestring, - - # arbitrary keys - Optional(basestring): late_bound(object) -}) - - -package_pod_schema = Schema(package_pod_schema_dict) - - -# ------------------------------------------------------------------------------ -# resource classes -# ------------------------------------------------------------------------------ - -class PackageRepositoryResource(Resource): - """Base class for all package-related resources. - - Attributes: - schema_error (`Exception`): Type of exception to throw on bad data. - repository_type (str): Type of package repository associated with this - resource type. - """ - schema_error = PackageMetadataError - repository_type = None - - @classmethod - def normalize_variables(cls, variables): - if "repository_type" not in variables or "location" not in \ - variables: - raise ResourceError("%s resources require a repository_type and " - "location" % cls.__name__) - return super(PackageRepositoryResource, cls).normalize_variables( - variables) - - def __init__(self, variables=None): - super(PackageRepositoryResource, self).__init__(variables) - - @cached_property - def uri(self): - return self._uri() - - @property - def location(self): - return self.get("location") - - @property - def name(self): - return self.get("name") - - def _uri(self): - """Return a URI. - - Implement this function to return a short, readable string that - uniquely identifies this resource. - """ - raise NotImplementedError - - -class PackageFamilyResource(PackageRepositoryResource): - """A package family. - - A repository implementation's package family resource(s) must derive from - this class. It must satisfy the schema `package_family_schema`. - """ - pass - - -class PackageResource(PackageRepositoryResource): - """A package. - - A repository implementation's package resource(s) must derive from this - class. It must satisfy the schema `package_schema`. - """ - - @classmethod - def normalize_variables(cls, variables): - """Make sure version is treated consistently - """ - # if the version is False, empty string, etc, throw it out - if variables.get('version', True) in ('', False, '_NO_VERSION', None): - del variables['version'] - return super(PackageResource, cls).normalize_variables(variables) - - @cached_property - def version(self): - ver_str = self.get("version", "") - return Version(ver_str) - - -class VariantResource(PackageResource): - """A package variant. - - A repository implementation's variant resource(s) must derive from this - class. It must satisfy the schema `variant_schema`. - - Even packages that do not have a 'variants' section contain a variant - in - this case it is the 'None' variant (the value of `index` is None). This - provides some internal consistency and simplifies the implementation. - """ - @property - def index(self): - return self.get("index", None) - - @cached_property - def root(self): - """Return the 'root' path of the variant.""" - return self._root() - - @cached_property - def subpath(self): - """Return the variant's 'subpath' - - The subpath is the relative path the variant's payload should be stored - under, relative to the package base. If None, implies that the variant - root matches the package base. - """ - return self._subpath() - - def _root(self, ignore_shortlinks=False): - raise NotImplementedError - - def _subpath(self, ignore_shortlinks=False): - raise NotImplementedError - - -# ------------------------------------------------------------------------------ -# resource helper classes -# -# Package repository plugins are not required to use the following classes, but -# they may help minimise the amount of code you need to write. -# ------------------------------------------------------------------------------ - -class PackageResourceHelper(PackageResource): - """PackageResource with some common functionality included. - """ - variant_key = None - - @cached_property - def commands(self): - return self._convert_to_rex(self._commands) - - @cached_property - def pre_commands(self): - return self._convert_to_rex(self._pre_commands) - - @cached_property - def post_commands(self): - return self._convert_to_rex(self._post_commands) - - def iter_variants(self): - num_variants = len(self.variants or []) - - if num_variants == 0: - indexes = [None] - else: - indexes = range(num_variants) - - for index in indexes: - variant = self._repository.get_resource( - self.variant_key, - location=self.location, - name=self.name, - version=self.get("version"), - index=index) - yield variant - - def _convert_to_rex(self, commands): - if isinstance(commands, list): - from rez.utils.backcompat import convert_old_commands - - msg = "package %r is using old-style commands." % self.uri - if config.disable_rez_1_compatibility or config.error_old_commands: - raise SchemaError(None, msg) - elif config.warn("old_commands"): - print_warning(msg) - commands = convert_old_commands(commands) - - if isinstance(commands, basestring): - return SourceCode(source=commands) - elif callable(commands): - return SourceCode(func=commands) - else: - return commands - -class _Metas(AttributeForwardMeta, LazyAttributeMeta): - pass - -class VariantResourceHelper(six.with_metaclass(_Metas, VariantResource)): - """Helper class for implementing variants that inherit properties from their - parent package. - - Since a variant overlaps so much with a package, here we use the forwarding - metaclass to forward our parent package's attributes onto ourself (with some - exceptions - eg 'variants', 'requires'). This is a common enough pattern - that it's supplied here for other repository plugins to use. - """ - - # Note: lazy key validation doesn't happen in this class, it just fowards on - # attributes from the package. But LazyAttributeMeta does still use this - # schema to create other class attributes, such as `validate_data`. - schema = variant_schema - - # forward Package attributes onto ourself - keys = schema_keys(package_schema) - set(["variants"]) - - def _uri(self): - index = self.index - idxstr = '' if index is None else str(index) - return "%s[%s]" % (self.parent.uri, idxstr) - - def _subpath(self, ignore_shortlinks=False): - if self.index is None: - return None - - if self.parent.hashed_variants: - vars_str = str(list(map(str, self.variant_requires))) - h = sha1(vars_str.encode("utf8")) - hashdir = h.hexdigest() - - if (not ignore_shortlinks) and \ - config.use_variant_shortlinks and \ - self.base is not None: - - # search for matching shortlink and use that - path = os.path.join(self.base, config.variant_shortlinks_dirname) - - if os.path.exists(path): - actual_root = os.path.join(self.base, hashdir) - linkname = find_matching_symlink(path, actual_root) - - if linkname: - return os.path.join( - config.variant_shortlinks_dirname, linkname) - - return hashdir - else: - dirs = [x.safe_str() for x in self.variant_requires] - subpath = os.path.join(*dirs) - return subpath - - def _root(self, ignore_shortlinks=False): - if self.base is None: - return None - elif self.index is None: - return self.base - else: - subpath = self._subpath(ignore_shortlinks=ignore_shortlinks) - root = os.path.join(self.base, subpath) - return root - - @cached_property - def variant_requires(self): - index = self.index - if index is None: - return [] - else: - try: - return self.parent.variants[index] or [] - except (IndexError, TypeError): - raise ResourceError( - "Unexpected error - variant %s cannot be found in its " - "parent package %s" % (self.uri, self.parent.uri)) - - @property - def wrapped(self): # forward Package attributes onto ourself - return self.parent - - def _load(self): - # doesn't have its own data, forwards on from parent instead - return None - - -# Copyright 2013-2016 Allan Johns. -# -# This library is free software: you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation, either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . diff --git a/src/rez/package_search.py b/src/rez/package_search.py index dd03602ed..f6ed6b68f 100644 --- a/src/rez/package_search.py +++ b/src/rez/package_search.py @@ -10,7 +10,7 @@ from collections import defaultdict import sys -from rez.packages_ import iter_package_families, iter_packages, get_latest_package +from rez.packages import iter_package_families, iter_packages, get_latest_package from rez.exceptions import PackageFamilyNotFoundError, ResourceContentError from rez.util import ProgressBar from rez.utils.colorize import critical, info, error, Printer diff --git a/src/rez/package_serialise.py b/src/rez/package_serialise.py index f7d6f046f..ffa4ce1af 100644 --- a/src/rez/package_serialise.py +++ b/src/rez/package_serialise.py @@ -2,7 +2,7 @@ from rez.vendor import yaml from rez.serialise import FileFormat -from rez.package_resources_ import help_schema, late_bound +from rez.package_resources import help_schema, late_bound from rez.vendor.schema.schema import Schema, Optional, And, Or, Use from rez.vendor.version.version import Version from rez.utils.schema import extensible_schema_dict diff --git a/src/rez/package_test.py b/src/rez/package_test.py index 485ed922a..7d64d7ce9 100644 --- a/src/rez/package_test.py +++ b/src/rez/package_test.py @@ -1,6 +1,6 @@ from rez.config import config from rez.resolved_context import ResolvedContext -from rez.packages_ import get_latest_package_from_string, Variant +from rez.packages import get_latest_package_from_string, Variant from rez.exceptions import RezError, PackageNotFoundError, PackageTestError from rez.utils.colorize import heading, Printer from rez.utils.logging_ import print_info, print_warning, print_error diff --git a/src/rez/packages.py b/src/rez/packages.py new file mode 100644 index 000000000..8a34ac42a --- /dev/null +++ b/src/rez/packages.py @@ -0,0 +1,792 @@ +from rez.package_repository import package_repository_manager +from rez.package_resources import PackageFamilyResource, PackageResource, \ + VariantResource, package_family_schema, package_schema, variant_schema, \ + package_release_keys, late_requires_schema +from rez.package_serialise import dump_package_data +from rez.utils import reraise +from rez.utils.sourcecode import SourceCode +from rez.utils.data_utils import cached_property +from rez.utils.formatting import StringFormatMixin, StringFormatType +from rez.utils.schema import schema_keys +from rez.utils.resources import ResourceHandle, ResourceWrapper +from rez.exceptions import PackageFamilyNotFoundError, ResourceError +from rez.vendor.version.version import VersionRange +from rez.vendor.version.requirement import VersionedObject +from rez.vendor.six import six +from rez.serialise import FileFormat +from rez.config import config +import sys + + +basestring = six.string_types[0] + +# ------------------------------------------------------------------------------ +# package-related classes +# ------------------------------------------------------------------------------ + +class PackageRepositoryResourceWrapper(ResourceWrapper, StringFormatMixin): + format_expand = StringFormatType.unchanged + + def validated_data(self): + data = ResourceWrapper.validated_data(self) + data = dict((k, v) for k, v in data.items() if v is not None) + return data + + @property + def repository(self): + """The package repository this resource comes from. + + Returns: + `PackageRepository`. + """ + return self.resource._repository + + +class PackageFamily(PackageRepositoryResourceWrapper): + """A package family. + + Note: + Do not instantiate this class directly, instead use the function + `iter_package_families`. + """ + keys = schema_keys(package_family_schema) + + def __init__(self, resource): + _check_class(resource, PackageFamilyResource) + super(PackageFamily, self).__init__(resource) + + def iter_packages(self): + """Iterate over the packages within this family, in no particular order. + + Returns: + `Package` iterator. + """ + for package in self.repository.iter_packages(self.resource): + yield Package(package) + + +class PackageBaseResourceWrapper(PackageRepositoryResourceWrapper): + """Abstract base class for `Package` and `Variant`. + """ + late_bind_schemas = { + "requires": late_requires_schema + } + + def __init__(self, resource, context=None): + super(PackageBaseResourceWrapper, self).__init__(resource) + self.context = context + + # cached results of late-bound funcs + self._late_binding_returnvalues = {} + + def set_context(self, context): + self.context = context + + def arbitrary_keys(self): + raise NotImplementedError + + @property + def uri(self): + return self.resource.uri + + @property + def config(self): + """Returns the config for this package. + + Defaults to global config if this package did not provide a 'config' + section. + """ + return self.resource.config or config + + @cached_property + def is_local(self): + """Returns True if the package is in the local package repository""" + local_repo = package_repository_manager.get_repository( + self.config.local_packages_path) + return (self.resource._repository.uid == local_repo.uid) + + def print_info(self, buf=None, format_=FileFormat.yaml, + skip_attributes=None, include_release=False): + """Print the contents of the package. + + Args: + buf (file-like object): Stream to write to. + format_ (`FileFormat`): Format to write in. + skip_attributes (list of str): List of attributes to not print. + include_release (bool): If True, include release-related attributes, + such as 'timestamp' and 'changelog' + """ + data = self.validated_data().copy() + + # config is a special case. We only really want to show any config settings + # that were in the package.py, not the entire Config contents that get + # grafted onto the Package/Variant instance. However Variant has an empy + # 'data' dict property, since it forwards data from its parent package. + data.pop("config", None) + if self.config: + if isinstance(self, Package): + config_dict = self.data.get("config") + else: + config_dict = self.parent.data.get("config") + data["config"] = config_dict + + if not include_release: + skip_attributes = list(skip_attributes or []) + list(package_release_keys) + + buf = buf or sys.stdout + dump_package_data(data, buf=buf, format_=format_, + skip_attributes=skip_attributes) + + def _wrap_forwarded(self, key, value): + if isinstance(value, SourceCode) and value.late_binding: + # get cached return value if present + value_ = self._late_binding_returnvalues.get(key, KeyError) + + if value_ is KeyError: + # evaluate the late-bound function + value_ = self._eval_late_binding(value) + + schema = self.late_bind_schemas.get(key) + if schema is not None: + value_ = schema.validate(value_) + + # cache result of late bound func + self._late_binding_returnvalues[key] = value_ + + return value_ + else: + return value + + def _eval_late_binding(self, sourcecode): + g = {} + + if self.context is None: + g["in_context"] = lambda: False + else: + g["in_context"] = lambda: True + g["context"] = self.context + + # 'request', 'system' etc + bindings = self.context._get_pre_resolve_bindings() + g.update(bindings) + + # Note that 'this' could be a `Package` or `Variant` instance. This is + # intentional; it just depends on how the package is accessed. + # + g["this"] = self + + # evaluate the late-bound function + sourcecode.set_package(self) + return sourcecode.exec_(globals_=g) + + +class Package(PackageBaseResourceWrapper): + """A package. + + Note: + Do not instantiate this class directly, instead use the function + `iter_packages` or `PackageFamily.iter_packages`. + """ + keys = schema_keys(package_schema) + + # This is to allow for a simple check like 'this.is_package' in late-bound + # funcs, where 'this' may be a package or variant. + # + is_package = True + is_variant = False + + def __init__(self, resource, context=None): + _check_class(resource, PackageResource) + super(Package, self).__init__(resource, context) + + # arbitrary keys + def __getattr__(self, name): + if name in self.data: + value = self.data[name] + return self._wrap_forwarded(name, value) + else: + raise AttributeError("Package instance has no attribute '%s'" % name) + + def arbitrary_keys(self): + """Get the arbitrary keys present in this package. + + These are any keys not in the standard list ('name', 'version' etc). + + Returns: + set of str: Arbitrary keys. + """ + return set(self.data.keys()) - set(self.keys) + + @cached_property + def qualified_name(self): + """Get the qualified name of the package. + + Returns: + str: Name of the package with version, eg "maya-2016.1". + """ + o = VersionedObject.construct(self.name, self.version) + return str(o) + + def as_exact_requirement(self): + """Get the package, as an exact requirement string. + + Returns: + Equivalent requirement string, eg "maya==2016.1" + """ + o = VersionedObject.construct(self.name, self.version) + return o.as_exact_requirement() + + @cached_property + def parent(self): + """Get the parent package family. + + Returns: + `PackageFamily`. + """ + family = self.repository.get_parent_package_family(self.resource) + return PackageFamily(family) if family else None + + @cached_property + def num_variants(self): + return len(self.data.get("variants", [])) + + @property + def is_relocatable(self): + """True if the package and its payload is safe to copy. + """ + if self.relocatable is None: + return config.default_relocatable + else: + return self.relocatable + + def iter_variants(self): + """Iterate over the variants within this package, in index order. + + Returns: + `Variant` iterator. + """ + for variant in self.repository.iter_variants(self.resource): + yield Variant(variant, context=self.context, parent=self) + + def get_variant(self, index=None): + """Get the variant with the associated index. + + Returns: + `Variant` object, or None if no variant with the given index exists. + """ + for variant in self.iter_variants(): + if variant.index == index: + return variant + + +class Variant(PackageBaseResourceWrapper): + """A package variant. + + Note: + Do not instantiate this class directly, instead use the function + `Package.iter_variants`. + """ + keys = schema_keys(variant_schema) + keys.update(["index", "root", "subpath"]) + + # See comment in `Package` + is_package = False + is_variant = True + + def __init__(self, resource, context=None, parent=None): + _check_class(resource, VariantResource) + super(Variant, self).__init__(resource, context) + self._parent = parent + + # arbitrary keys + def __getattr__(self, name): + try: + return self.parent.__getattr__(name) + except AttributeError: + raise AttributeError("Variant instance has no attribute '%s'" % name) + + def arbitrary_keys(self): + return self.parent.arbitrary_keys() + + @cached_property + def qualified_package_name(self): + o = VersionedObject.construct(self.name, self.version) + return str(o) + + @cached_property + def qualified_name(self): + """Get the qualified name of the variant. + + Returns: + str: Name of the variant with version and index, eg "maya-2016.1[1]". + """ + idxstr = '' if self.index is None else str(self.index) + return "%s[%s]" % (self.qualified_package_name, idxstr) + + @cached_property + def parent(self): + """Get the parent package. + + Returns: + `Package`. + """ + if self._parent is not None: + return self._parent + + try: + package = self.repository.get_parent_package(self.resource) + self._parent = Package(package, context=self.context) + except AttributeError as e: + reraise(e, ValueError) + + return self._parent + + @property + def variant_requires(self): + """Get the subset of requirements specific to this variant. + + Returns: + List of `Requirement` objects. + """ + if self.index is None: + return [] + else: + return self.parent.variants[self.index] or [] + + @property + def requires(self): + """Get variant requirements. + + This is a concatenation of the package requirements and those of this + specific variant. + + Returns: + List of `Requirement` objects. + """ + return ( + (self.parent.requires or []) + self.variant_requires + ) + + def get_requires(self, build_requires=False, private_build_requires=False): + """Get the requirements of the variant. + + Args: + build_requires (bool): If True, include build requirements. + private_build_requires (bool): If True, include private build + requirements. + + Returns: + List of `Requirement` objects. + """ + requires = self.requires or [] + + if build_requires: + requires = requires + (self.build_requires or []) + if private_build_requires: + requires = requires + (self.private_build_requires or []) + + return requires + + def install(self, path, dry_run=False, overrides=None): + """Install this variant into another package repository. + + If the package already exists, this variant will be correctly merged + into the package. If the variant already exists in this package, the + existing variant is returned. + + Args: + path (str): Path to destination package repository. + dry_run (bool): If True, do not actually install the variant. In this + mode, a `Variant` instance is only returned if the equivalent + variant already exists in this repository; otherwise, None is + returned. + overrides (dict): Use this to change or add attributes to the + installed variant. + + Returns: + `Variant` object - the (existing or newly created) variant in the + specified repository. If `dry_run` is True, None may be returned. + """ + repo = package_repository_manager.get_repository(path) + resource = repo.install_variant(self.resource, + dry_run=dry_run, + overrides=overrides) + if resource is None: + return None + elif resource is self.resource: + return self + else: + return Variant(resource) + + @property + def _non_shortlinked_subpath(self): + return self.resource._subpath(ignore_shortlinks=True) + + +class PackageSearchPath(object): + """A list of package repositories. + + For example, $REZ_PACKAGES_PATH refers to a list of repositories. + """ + def __init__(self, packages_path): + """Create a package repository list. + + Args: + packages_path (list of str): List of package repositories. + """ + self.paths = packages_path + + def iter_packages(self, name, range_=None): + """See `iter_packages`. + + Returns: + `Package` iterator. + """ + for package in iter_packages(name=name, range_=range_, paths=self.paths): + yield package + + def __contains__(self, package): + """See if a package is in this list of repositories. + + Note: + This does not verify the existance of the resource, only that the + resource's repository is in this list. + + Args: + package (`Package` or `Variant`): Package to search for. + + Returns: + bool: True if the resource is in the list of repositories, False + otherwise. + """ + return (package.resource._repository.uid in self._repository_uids) + + @cached_property + def _repository_uids(self): + uids = set() + for path in self.paths: + repo = package_repository_manager.get_repository(path) + uids.add(repo.uid) + return uids + + +# ------------------------------------------------------------------------------ +# resource acquisition functions +# ------------------------------------------------------------------------------ + +def iter_package_families(paths=None): + """Iterate over package families, in no particular order. + + Note that multiple package families with the same name can be returned. + Unlike packages, families later in the searchpath are not hidden by earlier + families. + + Args: + paths (list of str, optional): paths to search for package families, + defaults to `config.packages_path`. + + Returns: + `PackageFamily` iterator. + """ + for path in (paths or config.packages_path): + repo = package_repository_manager.get_repository(path) + for resource in repo.iter_package_families(): + yield PackageFamily(resource) + + +def iter_packages(name, range_=None, paths=None): + """Iterate over `Package` instances, in no particular order. + + Packages of the same name and version earlier in the search path take + precedence - equivalent packages later in the paths are ignored. Packages + are not returned in any specific order. + + Args: + name (str): Name of the package, eg 'maya'. + range_ (VersionRange or str): If provided, limits the versions returned + to those in `range_`. + paths (list of str, optional): paths to search for packages, defaults + to `config.packages_path`. + + Returns: + `Package` iterator. + """ + entries = _get_families(name, paths) + + seen = set() + for repo, family_resource in entries: + for package_resource in repo.iter_packages(family_resource): + key = (package_resource.name, package_resource.version) + if key in seen: + continue + + seen.add(key) + if range_: + if isinstance(range_, basestring): + range_ = VersionRange(range_) + if package_resource.version not in range_: + continue + + yield Package(package_resource) + + +def get_package(name, version, paths=None): + """Get an exact version of a package. + + Args: + name (str): Name of the package, eg 'maya'. + version (Version or str): Version of the package, eg '1.0.0' + paths (list of str, optional): paths to search for package, defaults + to `config.packages_path`. + + Returns: + `Package` object, or None if the package was not found. + """ + if isinstance(version, basestring): + range_ = VersionRange("==%s" % version) + else: + range_ = VersionRange.from_version(version, "==") + + it = iter_packages(name, range_, paths) + try: + return next(it) + except StopIteration: + return None + + +def get_package_from_handle(package_handle): + """Create a package given its handle (or serialized dict equivalent) + + Args: + package_handle (`ResourceHandle` or dict): Resource handle, or + equivalent serialized dict representation from + ResourceHandle.to_dict + + Returns: + `Package`. + """ + if isinstance(package_handle, dict): + package_handle = ResourceHandle.from_dict(package_handle) + package_resource = package_repository_manager.get_resource_from_handle(package_handle) + package = Package(package_resource) + return package + + +def get_package_from_string(txt, paths=None): + """Get a package given a string. + + Args: + txt (str): String such as 'foo', 'bah-1.3'. + paths (list of str, optional): paths to search for package, defaults + to `config.packages_path`. + + Returns: + `Package` instance, or None if no package was found. + """ + o = VersionedObject(txt) + return get_package(o.name, o.version, paths=paths) + + +def get_developer_package(path, format=None): + """Create a developer package. + + Args: + path (str): Path to dir containing package definition file. + format (str): Package definition file format, detected if None. + + Returns: + `DeveloperPackage`. + """ + from rez.developer_package import DeveloperPackage + return DeveloperPackage.from_path(path, format=format) + + +def create_package(name, data, package_cls=None): + """Create a package given package data. + + Args: + name (str): Package name. + data (dict): Package data. Must conform to `package_maker.package_schema`. + + Returns: + `Package` object. + """ + from rez.package_maker import PackageMaker + maker = PackageMaker(name, data, package_cls=package_cls) + return maker.get_package() + + +def get_variant(variant_handle, context=None): + """Create a variant given its handle (or serialized dict equivalent) + + Args: + variant_handle (`ResourceHandle` or dict): Resource handle, or + equivalent serialized dict representation from + ResourceHandle.to_dict + context (`ResolvedContext`): The context this variant is associated + with, if any. + + Returns: + `Variant`. + """ + if isinstance(variant_handle, dict): + variant_handle = ResourceHandle.from_dict(variant_handle) + + variant_resource = package_repository_manager.get_resource_from_handle(variant_handle) + variant = Variant(variant_resource, context=context) + return variant + + +def get_last_release_time(name, paths=None): + """Returns the most recent time this package was released. + + Note that releasing a variant into an already-released package is also + considered a package release. + + Returns: + int: Epoch time of last package release, or zero if this cannot be + determined. + """ + entries = _get_families(name, paths) + max_time = 0 + + for repo, family_resource in entries: + time_ = repo.get_last_release_time(family_resource) + if time_ == 0: + return 0 + max_time = max(max_time, time_) + return max_time + + +def get_completions(prefix, paths=None, family_only=False): + """Get autocompletion options given a prefix string. + + Example: + + >>> get_completions("may") + set(["maya", "maya_utils"]) + >>> get_completions("maya-") + set(["maya-2013.1", "maya-2015.0.sp1"]) + + Args: + prefix (str): Prefix to match. + paths (list of str): paths to search for packages, defaults to + `config.packages_path`. + family_only (bool): If True, only match package names, do not include + version component. + + Returns: + Set of strings, may be empty. + """ + op = None + if prefix: + if prefix[0] in ('!', '~'): + if family_only: + return set() + op = prefix[0] + prefix = prefix[1:] + + fam = None + for ch in ('-', '@', '#'): + if ch in prefix: + if family_only: + return set() + fam = prefix.split(ch)[0] + break + + words = set() + if not fam: + words = set(x.name for x in iter_package_families(paths=paths) + if x.name.startswith(prefix)) + if len(words) == 1: + fam = next(iter(words)) + + if family_only: + return words + + if fam: + it = iter_packages(fam, paths=paths) + words.update(x.qualified_name for x in it + if x.qualified_name.startswith(prefix)) + + if op: + words = set(op + x for x in words) + return words + + +def get_latest_package(name, range_=None, paths=None, error=False): + """Get the latest package for a given package name. + + Args: + name (str): Package name. + range_ (`VersionRange`): Version range to search within. + paths (list of str, optional): paths to search for package families, + defaults to `config.packages_path`. + error (bool): If True, raise an error if no package is found. + + Returns: + `Package` object, or None if no package is found. + """ + it = iter_packages(name, range_=range_, paths=paths) + try: + return max(it, key=lambda x: x.version) + except ValueError: # empty sequence + if error: + # FIXME this isn't correct, since the pkg fam may exist but a pkg + # in the range does not. + raise PackageFamilyNotFoundError("No such package family %r" % name) + return None + + +def get_latest_package_from_string(txt, paths=None, error=False): + """Get the latest package found within the given request string. + + Args: + txt (str): Request, eg 'foo-1.2+' + paths (list of str, optional): paths to search for package families, + defaults to `config.packages_path`. + error (bool): If True, raise an error if no package is found. + + Returns: + `Package` object, or None if no package is found. + """ + from rez.utils.formatting import PackageRequest + + req = PackageRequest(txt) + return get_latest_package(name=req.name, + range_=req.range_, + paths=paths, + error=error) + + +def _get_families(name, paths=None): + entries = [] + for path in (paths or config.packages_path): + repo = package_repository_manager.get_repository(path) + family_resource = repo.get_package_family(name) + if family_resource: + entries.append((repo, family_resource)) + + return entries + + +def _check_class(resource, cls): + if not isinstance(resource, cls): + raise ResourceError("Expected %s, got %s" + % (cls.__name__, resource.__class__.__name__)) + + +# Copyright 2013-2016 Allan Johns. +# +# This library is free software: you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation, either +# version 3 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . diff --git a/src/rez/packages_.py b/src/rez/packages_.py index b8f00c3ff..41d2878fe 100644 --- a/src/rez/packages_.py +++ b/src/rez/packages_.py @@ -1,792 +1,8 @@ -from rez.package_repository import package_repository_manager -from rez.package_resources_ import PackageFamilyResource, PackageResource, \ - VariantResource, package_family_schema, package_schema, variant_schema, \ - package_release_keys, late_requires_schema -from rez.package_serialise import dump_package_data -from rez.utils import reraise -from rez.utils.sourcecode import SourceCode -from rez.utils.data_utils import cached_property -from rez.utils.formatting import StringFormatMixin, StringFormatType -from rez.utils.schema import schema_keys -from rez.utils.resources import ResourceHandle, ResourceWrapper -from rez.exceptions import PackageFamilyNotFoundError, ResourceError -from rez.vendor.version.version import VersionRange -from rez.vendor.version.requirement import VersionedObject -from rez.vendor.six import six -from rez.serialise import FileFormat -from rez.config import config -import sys +import warnings +from rez.packages import * -basestring = six.string_types[0] - -# ------------------------------------------------------------------------------ -# package-related classes -# ------------------------------------------------------------------------------ - -class PackageRepositoryResourceWrapper(ResourceWrapper, StringFormatMixin): - format_expand = StringFormatType.unchanged - - def validated_data(self): - data = ResourceWrapper.validated_data(self) - data = dict((k, v) for k, v in data.items() if v is not None) - return data - - @property - def repository(self): - """The package repository this resource comes from. - - Returns: - `PackageRepository`. - """ - return self.resource._repository - - -class PackageFamily(PackageRepositoryResourceWrapper): - """A package family. - - Note: - Do not instantiate this class directly, instead use the function - `iter_package_families`. - """ - keys = schema_keys(package_family_schema) - - def __init__(self, resource): - _check_class(resource, PackageFamilyResource) - super(PackageFamily, self).__init__(resource) - - def iter_packages(self): - """Iterate over the packages within this family, in no particular order. - - Returns: - `Package` iterator. - """ - for package in self.repository.iter_packages(self.resource): - yield Package(package) - - -class PackageBaseResourceWrapper(PackageRepositoryResourceWrapper): - """Abstract base class for `Package` and `Variant`. - """ - late_bind_schemas = { - "requires": late_requires_schema - } - - def __init__(self, resource, context=None): - super(PackageBaseResourceWrapper, self).__init__(resource) - self.context = context - - # cached results of late-bound funcs - self._late_binding_returnvalues = {} - - def set_context(self, context): - self.context = context - - def arbitrary_keys(self): - raise NotImplementedError - - @property - def uri(self): - return self.resource.uri - - @property - def config(self): - """Returns the config for this package. - - Defaults to global config if this package did not provide a 'config' - section. - """ - return self.resource.config or config - - @cached_property - def is_local(self): - """Returns True if the package is in the local package repository""" - local_repo = package_repository_manager.get_repository( - self.config.local_packages_path) - return (self.resource._repository.uid == local_repo.uid) - - def print_info(self, buf=None, format_=FileFormat.yaml, - skip_attributes=None, include_release=False): - """Print the contents of the package. - - Args: - buf (file-like object): Stream to write to. - format_ (`FileFormat`): Format to write in. - skip_attributes (list of str): List of attributes to not print. - include_release (bool): If True, include release-related attributes, - such as 'timestamp' and 'changelog' - """ - data = self.validated_data().copy() - - # config is a special case. We only really want to show any config settings - # that were in the package.py, not the entire Config contents that get - # grafted onto the Package/Variant instance. However Variant has an empy - # 'data' dict property, since it forwards data from its parent package. - data.pop("config", None) - if self.config: - if isinstance(self, Package): - config_dict = self.data.get("config") - else: - config_dict = self.parent.data.get("config") - data["config"] = config_dict - - if not include_release: - skip_attributes = list(skip_attributes or []) + list(package_release_keys) - - buf = buf or sys.stdout - dump_package_data(data, buf=buf, format_=format_, - skip_attributes=skip_attributes) - - def _wrap_forwarded(self, key, value): - if isinstance(value, SourceCode) and value.late_binding: - # get cached return value if present - value_ = self._late_binding_returnvalues.get(key, KeyError) - - if value_ is KeyError: - # evaluate the late-bound function - value_ = self._eval_late_binding(value) - - schema = self.late_bind_schemas.get(key) - if schema is not None: - value_ = schema.validate(value_) - - # cache result of late bound func - self._late_binding_returnvalues[key] = value_ - - return value_ - else: - return value - - def _eval_late_binding(self, sourcecode): - g = {} - - if self.context is None: - g["in_context"] = lambda: False - else: - g["in_context"] = lambda: True - g["context"] = self.context - - # 'request', 'system' etc - bindings = self.context._get_pre_resolve_bindings() - g.update(bindings) - - # Note that 'this' could be a `Package` or `Variant` instance. This is - # intentional; it just depends on how the package is accessed. - # - g["this"] = self - - # evaluate the late-bound function - sourcecode.set_package(self) - return sourcecode.exec_(globals_=g) - - -class Package(PackageBaseResourceWrapper): - """A package. - - Note: - Do not instantiate this class directly, instead use the function - `iter_packages` or `PackageFamily.iter_packages`. - """ - keys = schema_keys(package_schema) - - # This is to allow for a simple check like 'this.is_package' in late-bound - # funcs, where 'this' may be a package or variant. - # - is_package = True - is_variant = False - - def __init__(self, resource, context=None): - _check_class(resource, PackageResource) - super(Package, self).__init__(resource, context) - - # arbitrary keys - def __getattr__(self, name): - if name in self.data: - value = self.data[name] - return self._wrap_forwarded(name, value) - else: - raise AttributeError("Package instance has no attribute '%s'" % name) - - def arbitrary_keys(self): - """Get the arbitrary keys present in this package. - - These are any keys not in the standard list ('name', 'version' etc). - - Returns: - set of str: Arbitrary keys. - """ - return set(self.data.keys()) - set(self.keys) - - @cached_property - def qualified_name(self): - """Get the qualified name of the package. - - Returns: - str: Name of the package with version, eg "maya-2016.1". - """ - o = VersionedObject.construct(self.name, self.version) - return str(o) - - def as_exact_requirement(self): - """Get the package, as an exact requirement string. - - Returns: - Equivalent requirement string, eg "maya==2016.1" - """ - o = VersionedObject.construct(self.name, self.version) - return o.as_exact_requirement() - - @cached_property - def parent(self): - """Get the parent package family. - - Returns: - `PackageFamily`. - """ - family = self.repository.get_parent_package_family(self.resource) - return PackageFamily(family) if family else None - - @cached_property - def num_variants(self): - return len(self.data.get("variants", [])) - - @property - def is_relocatable(self): - """True if the package and its payload is safe to copy. - """ - if self.relocatable is None: - return config.default_relocatable - else: - return self.relocatable - - def iter_variants(self): - """Iterate over the variants within this package, in index order. - - Returns: - `Variant` iterator. - """ - for variant in self.repository.iter_variants(self.resource): - yield Variant(variant, context=self.context, parent=self) - - def get_variant(self, index=None): - """Get the variant with the associated index. - - Returns: - `Variant` object, or None if no variant with the given index exists. - """ - for variant in self.iter_variants(): - if variant.index == index: - return variant - - -class Variant(PackageBaseResourceWrapper): - """A package variant. - - Note: - Do not instantiate this class directly, instead use the function - `Package.iter_variants`. - """ - keys = schema_keys(variant_schema) - keys.update(["index", "root", "subpath"]) - - # See comment in `Package` - is_package = False - is_variant = True - - def __init__(self, resource, context=None, parent=None): - _check_class(resource, VariantResource) - super(Variant, self).__init__(resource, context) - self._parent = parent - - # arbitrary keys - def __getattr__(self, name): - try: - return self.parent.__getattr__(name) - except AttributeError: - raise AttributeError("Variant instance has no attribute '%s'" % name) - - def arbitrary_keys(self): - return self.parent.arbitrary_keys() - - @cached_property - def qualified_package_name(self): - o = VersionedObject.construct(self.name, self.version) - return str(o) - - @cached_property - def qualified_name(self): - """Get the qualified name of the variant. - - Returns: - str: Name of the variant with version and index, eg "maya-2016.1[1]". - """ - idxstr = '' if self.index is None else str(self.index) - return "%s[%s]" % (self.qualified_package_name, idxstr) - - @cached_property - def parent(self): - """Get the parent package. - - Returns: - `Package`. - """ - if self._parent is not None: - return self._parent - - try: - package = self.repository.get_parent_package(self.resource) - self._parent = Package(package, context=self.context) - except AttributeError as e: - reraise(e, ValueError) - - return self._parent - - @property - def variant_requires(self): - """Get the subset of requirements specific to this variant. - - Returns: - List of `Requirement` objects. - """ - if self.index is None: - return [] - else: - return self.parent.variants[self.index] or [] - - @property - def requires(self): - """Get variant requirements. - - This is a concatenation of the package requirements and those of this - specific variant. - - Returns: - List of `Requirement` objects. - """ - return ( - (self.parent.requires or []) + self.variant_requires - ) - - def get_requires(self, build_requires=False, private_build_requires=False): - """Get the requirements of the variant. - - Args: - build_requires (bool): If True, include build requirements. - private_build_requires (bool): If True, include private build - requirements. - - Returns: - List of `Requirement` objects. - """ - requires = self.requires or [] - - if build_requires: - requires = requires + (self.build_requires or []) - if private_build_requires: - requires = requires + (self.private_build_requires or []) - - return requires - - def install(self, path, dry_run=False, overrides=None): - """Install this variant into another package repository. - - If the package already exists, this variant will be correctly merged - into the package. If the variant already exists in this package, the - existing variant is returned. - - Args: - path (str): Path to destination package repository. - dry_run (bool): If True, do not actually install the variant. In this - mode, a `Variant` instance is only returned if the equivalent - variant already exists in this repository; otherwise, None is - returned. - overrides (dict): Use this to change or add attributes to the - installed variant. - - Returns: - `Variant` object - the (existing or newly created) variant in the - specified repository. If `dry_run` is True, None may be returned. - """ - repo = package_repository_manager.get_repository(path) - resource = repo.install_variant(self.resource, - dry_run=dry_run, - overrides=overrides) - if resource is None: - return None - elif resource is self.resource: - return self - else: - return Variant(resource) - - @property - def _non_shortlinked_subpath(self): - return self.resource._subpath(ignore_shortlinks=True) - - -class PackageSearchPath(object): - """A list of package repositories. - - For example, $REZ_PACKAGES_PATH refers to a list of repositories. - """ - def __init__(self, packages_path): - """Create a package repository list. - - Args: - packages_path (list of str): List of package repositories. - """ - self.paths = packages_path - - def iter_packages(self, name, range_=None): - """See `iter_packages`. - - Returns: - `Package` iterator. - """ - for package in iter_packages(name=name, range_=range_, paths=self.paths): - yield package - - def __contains__(self, package): - """See if a package is in this list of repositories. - - Note: - This does not verify the existance of the resource, only that the - resource's repository is in this list. - - Args: - package (`Package` or `Variant`): Package to search for. - - Returns: - bool: True if the resource is in the list of repositories, False - otherwise. - """ - return (package.resource._repository.uid in self._repository_uids) - - @cached_property - def _repository_uids(self): - uids = set() - for path in self.paths: - repo = package_repository_manager.get_repository(path) - uids.add(repo.uid) - return uids - - -# ------------------------------------------------------------------------------ -# resource acquisition functions -# ------------------------------------------------------------------------------ - -def iter_package_families(paths=None): - """Iterate over package families, in no particular order. - - Note that multiple package families with the same name can be returned. - Unlike packages, families later in the searchpath are not hidden by earlier - families. - - Args: - paths (list of str, optional): paths to search for package families, - defaults to `config.packages_path`. - - Returns: - `PackageFamily` iterator. - """ - for path in (paths or config.packages_path): - repo = package_repository_manager.get_repository(path) - for resource in repo.iter_package_families(): - yield PackageFamily(resource) - - -def iter_packages(name, range_=None, paths=None): - """Iterate over `Package` instances, in no particular order. - - Packages of the same name and version earlier in the search path take - precedence - equivalent packages later in the paths are ignored. Packages - are not returned in any specific order. - - Args: - name (str): Name of the package, eg 'maya'. - range_ (VersionRange or str): If provided, limits the versions returned - to those in `range_`. - paths (list of str, optional): paths to search for packages, defaults - to `config.packages_path`. - - Returns: - `Package` iterator. - """ - entries = _get_families(name, paths) - - seen = set() - for repo, family_resource in entries: - for package_resource in repo.iter_packages(family_resource): - key = (package_resource.name, package_resource.version) - if key in seen: - continue - - seen.add(key) - if range_: - if isinstance(range_, basestring): - range_ = VersionRange(range_) - if package_resource.version not in range_: - continue - - yield Package(package_resource) - - -def get_package(name, version, paths=None): - """Get an exact version of a package. - - Args: - name (str): Name of the package, eg 'maya'. - version (Version or str): Version of the package, eg '1.0.0' - paths (list of str, optional): paths to search for package, defaults - to `config.packages_path`. - - Returns: - `Package` object, or None if the package was not found. - """ - if isinstance(version, basestring): - range_ = VersionRange("==%s" % version) - else: - range_ = VersionRange.from_version(version, "==") - - it = iter_packages(name, range_, paths) - try: - return next(it) - except StopIteration: - return None - - -def get_package_from_handle(package_handle): - """Create a package given its handle (or serialized dict equivalent) - - Args: - package_handle (`ResourceHandle` or dict): Resource handle, or - equivalent serialized dict representation from - ResourceHandle.to_dict - - Returns: - `Package`. - """ - if isinstance(package_handle, dict): - package_handle = ResourceHandle.from_dict(package_handle) - package_resource = package_repository_manager.get_resource_from_handle(package_handle) - package = Package(package_resource) - return package - - -def get_package_from_string(txt, paths=None): - """Get a package given a string. - - Args: - txt (str): String such as 'foo', 'bah-1.3'. - paths (list of str, optional): paths to search for package, defaults - to `config.packages_path`. - - Returns: - `Package` instance, or None if no package was found. - """ - o = VersionedObject(txt) - return get_package(o.name, o.version, paths=paths) - - -def get_developer_package(path, format=None): - """Create a developer package. - - Args: - path (str): Path to dir containing package definition file. - format (str): Package definition file format, detected if None. - - Returns: - `DeveloperPackage`. - """ - from rez.developer_package import DeveloperPackage - return DeveloperPackage.from_path(path, format=format) - - -def create_package(name, data, package_cls=None): - """Create a package given package data. - - Args: - name (str): Package name. - data (dict): Package data. Must conform to `package_maker.package_schema`. - - Returns: - `Package` object. - """ - from rez.package_maker__ import PackageMaker - maker = PackageMaker(name, data, package_cls=package_cls) - return maker.get_package() - - -def get_variant(variant_handle, context=None): - """Create a variant given its handle (or serialized dict equivalent) - - Args: - variant_handle (`ResourceHandle` or dict): Resource handle, or - equivalent serialized dict representation from - ResourceHandle.to_dict - context (`ResolvedContext`): The context this variant is associated - with, if any. - - Returns: - `Variant`. - """ - if isinstance(variant_handle, dict): - variant_handle = ResourceHandle.from_dict(variant_handle) - - variant_resource = package_repository_manager.get_resource_from_handle(variant_handle) - variant = Variant(variant_resource, context=context) - return variant - - -def get_last_release_time(name, paths=None): - """Returns the most recent time this package was released. - - Note that releasing a variant into an already-released package is also - considered a package release. - - Returns: - int: Epoch time of last package release, or zero if this cannot be - determined. - """ - entries = _get_families(name, paths) - max_time = 0 - - for repo, family_resource in entries: - time_ = repo.get_last_release_time(family_resource) - if time_ == 0: - return 0 - max_time = max(max_time, time_) - return max_time - - -def get_completions(prefix, paths=None, family_only=False): - """Get autocompletion options given a prefix string. - - Example: - - >>> get_completions("may") - set(["maya", "maya_utils"]) - >>> get_completions("maya-") - set(["maya-2013.1", "maya-2015.0.sp1"]) - - Args: - prefix (str): Prefix to match. - paths (list of str): paths to search for packages, defaults to - `config.packages_path`. - family_only (bool): If True, only match package names, do not include - version component. - - Returns: - Set of strings, may be empty. - """ - op = None - if prefix: - if prefix[0] in ('!', '~'): - if family_only: - return set() - op = prefix[0] - prefix = prefix[1:] - - fam = None - for ch in ('-', '@', '#'): - if ch in prefix: - if family_only: - return set() - fam = prefix.split(ch)[0] - break - - words = set() - if not fam: - words = set(x.name for x in iter_package_families(paths=paths) - if x.name.startswith(prefix)) - if len(words) == 1: - fam = next(iter(words)) - - if family_only: - return words - - if fam: - it = iter_packages(fam, paths=paths) - words.update(x.qualified_name for x in it - if x.qualified_name.startswith(prefix)) - - if op: - words = set(op + x for x in words) - return words - - -def get_latest_package(name, range_=None, paths=None, error=False): - """Get the latest package for a given package name. - - Args: - name (str): Package name. - range_ (`VersionRange`): Version range to search within. - paths (list of str, optional): paths to search for package families, - defaults to `config.packages_path`. - error (bool): If True, raise an error if no package is found. - - Returns: - `Package` object, or None if no package is found. - """ - it = iter_packages(name, range_=range_, paths=paths) - try: - return max(it, key=lambda x: x.version) - except ValueError: # empty sequence - if error: - # FIXME this isn't correct, since the pkg fam may exist but a pkg - # in the range does not. - raise PackageFamilyNotFoundError("No such package family %r" % name) - return None - - -def get_latest_package_from_string(txt, paths=None, error=False): - """Get the latest package found within the given request string. - - Args: - txt (str): Request, eg 'foo-1.2+' - paths (list of str, optional): paths to search for package families, - defaults to `config.packages_path`. - error (bool): If True, raise an error if no package is found. - - Returns: - `Package` object, or None if no package is found. - """ - from rez.utils.formatting import PackageRequest - - req = PackageRequest(txt) - return get_latest_package(name=req.name, - range_=req.range_, - paths=paths, - error=error) - - -def _get_families(name, paths=None): - entries = [] - for path in (paths or config.packages_path): - repo = package_repository_manager.get_repository(path) - family_resource = repo.get_package_family(name) - if family_resource: - entries.append((repo, family_resource)) - - return entries - - -def _check_class(resource, cls): - if not isinstance(resource, cls): - raise ResourceError("Expected %s, got %s" - % (cls.__name__, resource.__class__.__name__)) - - -# Copyright 2013-2016 Allan Johns. -# -# This library is free software: you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation, either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . +warnings.warn( + "rez.packages_ is deprecated; import rez.packages instead", + DeprecationWarning +) diff --git a/src/rez/pip.py b/src/rez/pip.py index cadea8b3a..a0db31756 100644 --- a/src/rez/pip.py +++ b/src/rez/pip.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -from rez.packages_ import get_latest_package +from rez.packages import get_latest_package from rez.vendor.version.version import Version, VersionError from rez.vendor.distlib import DistlibException from rez.vendor.distlib.database import DistributionPath @@ -15,7 +15,7 @@ from rez.utils.logging_ import print_debug, print_info, print_warning from rez.exceptions import BuildError, PackageFamilyNotFoundError, \ PackageNotFoundError, RezSystemError, convert_errors -from rez.package_maker__ import make_package +from rez.package_maker import make_package from rez.config import config from rez.system import System from rez.utils.platform_ import platform_ diff --git a/src/rez/release_hook.py b/src/rez/release_hook.py index 462018a3d..11cd2295d 100644 --- a/src/rez/release_hook.py +++ b/src/rez/release_hook.py @@ -1,5 +1,5 @@ from rez.utils.logging_ import print_warning, print_debug -from rez.packages_ import get_developer_package +from rez.packages import get_developer_package from rez.vendor.enum import Enum diff --git a/src/rez/release_vcs.py b/src/rez/release_vcs.py index 8a8ecdfab..18d5950df 100644 --- a/src/rez/release_vcs.py +++ b/src/rez/release_vcs.py @@ -1,5 +1,5 @@ from rez.exceptions import ReleaseVCSError -from rez.packages_ import get_developer_package +from rez.packages import get_developer_package from rez.util import which from rez.utils.execution import Popen from rez.utils.logging_ import print_debug diff --git a/src/rez/resolved_context.py b/src/rez/resolved_context.py index 6e2386b46..b376bfefd 100644 --- a/src/rez/resolved_context.py +++ b/src/rez/resolved_context.py @@ -19,7 +19,7 @@ from rez.rex_bindings import VersionBinding, VariantBinding, \ VariantsBinding, RequirementsBinding from rez import package_order -from rez.packages_ import get_variant, iter_packages +from rez.packages import get_variant, iter_packages from rez.package_filter import PackageFilterList from rez.shells import create_shell from rez.exceptions import ResolvedContextError, PackageCommandError, RezError diff --git a/src/rez/resolver.py b/src/rez/resolver.py index 0fd254ed4..f72b918d6 100644 --- a/src/rez/resolver.py +++ b/src/rez/resolver.py @@ -1,6 +1,6 @@ from rez.solver import Solver, SolverStatus, PackageVariantCache from rez.package_repository import package_repository_manager -from rez.packages_ import get_variant, get_last_release_time +from rez.packages import get_variant, get_last_release_time from rez.package_filter import PackageFilterList, TimestampRule from rez.utils.memcached import memcached_client, pool_memcached_connections from rez.utils.logging_ import log_duration diff --git a/src/rez/serialise.py b/src/rez/serialise.py index 6b0959ab6..ecbfc9a96 100644 --- a/src/rez/serialise.py +++ b/src/rez/serialise.py @@ -9,7 +9,7 @@ import os.path import threading -from rez.package_resources_ import package_rex_keys +from rez.package_resources import package_rex_keys from rez.utils.scope import ScopeContext from rez.utils.sourcecode import SourceCode, early, late, include from rez.utils.filesystem import TempDirs diff --git a/src/rez/solver.py b/src/rez/solver.py index 25b06c7ba..6bd3641c7 100644 --- a/src/rez/solver.py +++ b/src/rez/solver.py @@ -10,7 +10,7 @@ from __future__ import print_function from rez.config import config -from rez.packages_ import iter_packages +from rez.packages import iter_packages from rez.package_repository import package_repo_stats from rez.utils.logging_ import print_debug from rez.utils.data_utils import cached_property diff --git a/src/rez/status.py b/src/rez/status.py index 66f966e97..99d3b19b8 100644 --- a/src/rez/status.py +++ b/src/rez/status.py @@ -7,7 +7,7 @@ from rez import __version__ from rez.utils.data_utils import cached_property from rez.resolved_context import ResolvedContext -from rez.packages_ import iter_packages, Package +from rez.packages import iter_packages, Package from rez.suite import Suite from rez.wrapper import Wrapper from rez.utils.colorize import local, warning, critical, Printer diff --git a/src/rez/tests/test_build.py b/src/rez/tests/test_build.py index 3ac7a8b55..220c753ae 100644 --- a/src/rez/tests/test_build.py +++ b/src/rez/tests/test_build.py @@ -1,7 +1,7 @@ """ test the build system """ -from rez.build_process_ import create_build_process +from rez.build_process import create_build_process from rez.build_system import create_build_system from rez.resolved_context import ResolvedContext from rez.exceptions import BuildError, BuildContextResolveError,\ diff --git a/src/rez/tests/test_completion.py b/src/rez/tests/test_completion.py index 77d614290..13ce77b40 100644 --- a/src/rez/tests/test_completion.py +++ b/src/rez/tests/test_completion.py @@ -4,7 +4,7 @@ import unittest from rez.tests.util import TestBase from rez.config import Config, get_module_root_config -from rez.packages_ import get_completions +from rez.packages import get_completions import os import os.path diff --git a/src/rez/tests/test_config.py b/src/rez/tests/test_config.py index b11b2e621..2e8f86136 100644 --- a/src/rez/tests/test_config.py +++ b/src/rez/tests/test_config.py @@ -7,7 +7,7 @@ from rez.config import Config, get_module_root_config, _replace_config from rez.system import system from rez.utils.data_utils import RO_AttrDictWrapper -from rez.packages_ import get_developer_package +from rez.packages import get_developer_package import os import os.path diff --git a/src/rez/tests/test_copy_package.py b/src/rez/tests/test_copy_package.py index f1bfb18cc..73a687594 100644 --- a/src/rez/tests/test_copy_package.py +++ b/src/rez/tests/test_copy_package.py @@ -7,10 +7,10 @@ import os from rez.system import system -from rez.build_process_ import create_build_process +from rez.build_process import create_build_process from rez.build_system import create_build_system from rez.resolved_context import ResolvedContext -from rez.packages_ import get_latest_package +from rez.packages import get_latest_package from rez.package_copy import copy_package from rez.vendor.version.version import VersionRange from rez.tests.util import TestBase, TempdirMixin diff --git a/src/rez/tests/test_imports.py b/src/rez/tests/test_imports.py index 95957b04e..db79fd854 100644 --- a/src/rez/tests/test_imports.py +++ b/src/rez/tests/test_imports.py @@ -9,17 +9,17 @@ class TestImports(TestBase): def test_1(self): """import every file in rez.""" import rez - import rez.build_process_ + import rez.build_process import rez.build_system import rez.config import rez.exceptions import rez.package_help - import rez.package_maker__ + import rez.package_maker import rez.package_repository - import rez.package_resources_ + import rez.package_resources import rez.package_search import rez.package_serialise - import rez.packages_ + import rez.packages import rez.plugin_managers import rez.release_hook import rez.release_vcs diff --git a/src/rez/tests/test_packages.py b/src/rez/tests/test_packages.py index 4fc798679..342b2e5e1 100644 --- a/src/rez/tests/test_packages.py +++ b/src/rez/tests/test_packages.py @@ -1,11 +1,11 @@ """ test package iteration, serialization etc """ -from rez.packages_ import iter_package_families, iter_packages, get_package, \ +from rez.packages import iter_package_families, iter_packages, get_package, \ create_package, get_developer_package from rez.package_py_utils import expand_requirement from rez.package_repository import create_memory_package_repository -from rez.package_resources_ import package_release_keys +from rez.package_resources import package_release_keys from rez.tests.util import TestBase, TempdirMixin from rez.utils.formatting import PackageRequest from rez.utils.platform_ import platform_ diff --git a/src/rez/tests/test_release.py b/src/rez/tests/test_release.py index 24a9d4202..38a41107b 100644 --- a/src/rez/tests/test_release.py +++ b/src/rez/tests/test_release.py @@ -1,11 +1,11 @@ """ test the release system """ -from rez.build_process_ import create_build_process +from rez.build_process import create_build_process from rez.build_system import create_build_system from rez.resolved_context import ResolvedContext from rez.release_vcs import create_release_vcs -from rez.packages_ import iter_packages +from rez.packages import iter_packages from rez.vendor import yaml from rez.system import system from rez.exceptions import ReleaseError, ReleaseVCSError diff --git a/src/rez/utils/_version.py b/src/rez/utils/_version.py index 009b02d06..0fb9efced 100644 --- a/src/rez/utils/_version.py +++ b/src/rez/utils/_version.py @@ -1,7 +1,7 @@ # Update this value to version up Rez. Do not place anything else in this file. -_rez_version = "2.52.0" +_rez_version = "2.52.1" # Copyright 2013-2016 Allan Johns. diff --git a/src/rez/utils/diff_packages.py b/src/rez/utils/diff_packages.py index c2f954e3d..9306b3289 100644 --- a/src/rez/utils/diff_packages.py +++ b/src/rez/utils/diff_packages.py @@ -1,6 +1,6 @@ from __future__ import print_function -from rez.packages_ import iter_packages +from rez.packages import iter_packages from rez.config import config from rez.plugin_managers import plugin_manager from rez.exceptions import RezError diff --git a/src/rez/utils/sourcecode.py b/src/rez/utils/sourcecode.py index 61f3883bd..508f6a1d6 100644 --- a/src/rez/utils/sourcecode.py +++ b/src/rez/utils/sourcecode.py @@ -32,7 +32,7 @@ def late(): this decorator - otherwise it is understood that you want your attribute to be a function, not the return value of that function. """ - from rez.package_resources_ import package_rex_keys + from rez.package_resources import package_rex_keys def decorated(fn): diff --git a/src/rez/wrapper.py b/src/rez/wrapper.py index 23b9f0479..9b4e4f32a 100644 --- a/src/rez/wrapper.py +++ b/src/rez/wrapper.py @@ -229,7 +229,7 @@ def print_package_versions(self): self._print_conflicting(variants) return 1 else: - from rez.packages_ import iter_packages + from rez.packages import iter_packages variant = next(iter(variants)) it = iter_packages(name=variant.name) rows = [] diff --git a/src/rezgui/objects/LoadPackagesThread.py b/src/rezgui/objects/LoadPackagesThread.py index 0aa333946..22e2067f4 100644 --- a/src/rezgui/objects/LoadPackagesThread.py +++ b/src/rezgui/objects/LoadPackagesThread.py @@ -1,5 +1,5 @@ from Qt import QtCore -from rez.packages_ import iter_packages +from rez.packages import iter_packages class LoadPackagesThread(QtCore.QObject): diff --git a/src/rezgui/widgets/ContextTableWidget.py b/src/rezgui/widgets/ContextTableWidget.py index e649eea82..0e1aa07ed 100644 --- a/src/rezgui/widgets/ContextTableWidget.py +++ b/src/rezgui/widgets/ContextTableWidget.py @@ -7,7 +7,7 @@ from rezgui.mixins.ContextViewMixin import ContextViewMixin from rezgui.models.ContextModel import ContextModel from rezgui.objects.App import app -from rez.packages_ import iter_packages +from rez.packages import iter_packages from rez.vendor.version.requirement import Requirement from rez.vendor.version.version import VersionRange from functools import partial diff --git a/src/rezgui/widgets/PackageLineEdit.py b/src/rezgui/widgets/PackageLineEdit.py index db450a537..46f1ad593 100644 --- a/src/rezgui/widgets/PackageLineEdit.py +++ b/src/rezgui/widgets/PackageLineEdit.py @@ -1,7 +1,7 @@ from Qt import QtCore, QtWidgets, QtGui from rezgui.models.ContextModel import ContextModel from rezgui.mixins.ContextViewMixin import ContextViewMixin -from rez.packages_ import get_completions, iter_packages +from rez.packages import get_completions, iter_packages from rez.vendor.version.requirement import Requirement diff --git a/src/rezgui/widgets/PackageTabWidget.py b/src/rezgui/widgets/PackageTabWidget.py index 416b58f1e..e350d5f38 100644 --- a/src/rezgui/widgets/PackageTabWidget.py +++ b/src/rezgui/widgets/PackageTabWidget.py @@ -9,7 +9,7 @@ from rezgui.widgets.VariantDetailsWidget import VariantDetailsWidget from rezgui.widgets.VariantsList import VariantsList -from rez.packages_ import Package, Variant +from rez.packages import Package, Variant class PackageTabWidget(QtWidgets.QTabWidget, ContextViewMixin): diff --git a/src/rezgui/widgets/PackageVersionsTable.py b/src/rezgui/widgets/PackageVersionsTable.py index df8f66a9b..8ee09e557 100644 --- a/src/rezgui/widgets/PackageVersionsTable.py +++ b/src/rezgui/widgets/PackageVersionsTable.py @@ -2,7 +2,7 @@ from rezgui.mixins.ContextViewMixin import ContextViewMixin from rezgui.models.ContextModel import ContextModel from rezgui.util import get_timestamp_str -from rez.packages_ import iter_packages +from rez.packages import iter_packages from rez.exceptions import RezError diff --git a/src/rezgui/widgets/VariantCellWidget.py b/src/rezgui/widgets/VariantCellWidget.py index 96a6ff401..3e3f2dbe1 100644 --- a/src/rezgui/widgets/VariantCellWidget.py +++ b/src/rezgui/widgets/VariantCellWidget.py @@ -2,7 +2,7 @@ from rezgui.util import create_pane, get_icon_widget, add_menu_action, update_font from rezgui.models.ContextModel import ContextModel from rezgui.mixins.ContextViewMixin import ContextViewMixin -from rez.packages_ import PackageSearchPath +from rez.packages import PackageSearchPath from rez.package_filter import PackageFilterList from rez.resolved_context import PatchLock, get_lock_request from rez.vendor.version.requirement import RequirementList diff --git a/src/rezgui/widgets/VariantSummaryWidget.py b/src/rezgui/widgets/VariantSummaryWidget.py index 17380d01c..987f747c4 100644 --- a/src/rezgui/widgets/VariantSummaryWidget.py +++ b/src/rezgui/widgets/VariantSummaryWidget.py @@ -1,6 +1,6 @@ from Qt import QtCore, QtWidgets from rezgui.util import create_pane, get_timestamp_str -from rez.packages_ import Package, Variant +from rez.packages import Package, Variant from rez.util import find_last_sublist diff --git a/src/rezgui/widgets/VariantVersionsTable.py b/src/rezgui/widgets/VariantVersionsTable.py index aed6d3334..d532f4aaf 100644 --- a/src/rezgui/widgets/VariantVersionsTable.py +++ b/src/rezgui/widgets/VariantVersionsTable.py @@ -2,7 +2,7 @@ from rezgui.mixins.ContextViewMixin import ContextViewMixin from rez.package_filter import PackageFilterList from rezgui.util import get_timestamp_str, update_font, get_icon_widget, create_pane -from rez.packages_ import iter_packages +from rez.packages import iter_packages from rez.vendor.version.version import VersionRange diff --git a/src/rezgui/widgets/VariantsList.py b/src/rezgui/widgets/VariantsList.py index 218017311..0bda5a944 100644 --- a/src/rezgui/widgets/VariantsList.py +++ b/src/rezgui/widgets/VariantsList.py @@ -1,5 +1,5 @@ from Qt import QtCore, QtWidgets -from rez.packages_ import Package +from rez.packages import Package class VariantsList(QtWidgets.QTableWidget): diff --git a/src/rezplugins/build_process/local.py b/src/rezplugins/build_process/local.py index 5264767b6..82520dd92 100644 --- a/src/rezplugins/build_process/local.py +++ b/src/rezplugins/build_process/local.py @@ -3,7 +3,7 @@ """ from rez.config import config from rez.package_repository import package_repository_manager -from rez.build_process_ import BuildProcessHelper, BuildType +from rez.build_process import BuildProcessHelper, BuildType from rez.release_hook import ReleaseHookEvent from rez.exceptions import BuildError, PackageTestError from rez.utils import with_noop diff --git a/src/rezplugins/build_process/remote.py b/src/rezplugins/build_process/remote.py index 7da40c190..f459e04c3 100644 --- a/src/rezplugins/build_process/remote.py +++ b/src/rezplugins/build_process/remote.py @@ -1,7 +1,7 @@ """ Builds packages on remote hosts """ -from rez.build_process_ import BuildProcessHelper +from rez.build_process import BuildProcessHelper from rez.exceptions import BuildError diff --git a/src/rezplugins/build_system/bez.py b/src/rezplugins/build_system/bez.py index 394684d65..a0bb0d163 100644 --- a/src/rezplugins/build_system/bez.py +++ b/src/rezplugins/build_system/bez.py @@ -2,9 +2,9 @@ Built-in simple python build system """ from rez.build_system import BuildSystem -from rez.build_process_ import BuildType +from rez.build_process import BuildType from rez.utils.execution import create_forwarding_script -from rez.packages_ import get_developer_package +from rez.packages import get_developer_package from rez.resolved_context import ResolvedContext from rez.config import config from rez.utils.yaml import dump_yaml diff --git a/src/rezplugins/build_system/cmake.py b/src/rezplugins/build_system/cmake.py index fbbe2f8b1..f67b0b97c 100644 --- a/src/rezplugins/build_system/cmake.py +++ b/src/rezplugins/build_system/cmake.py @@ -4,11 +4,11 @@ from __future__ import print_function from rez.build_system import BuildSystem -from rez.build_process_ import BuildType +from rez.build_process import BuildType from rez.resolved_context import ResolvedContext from rez.exceptions import BuildSystemError from rez.utils.execution import create_forwarding_script -from rez.packages_ import get_developer_package +from rez.packages import get_developer_package from rez.utils.platform_ import platform_ from rez.config import config from rez.backport.shutilwhich import which diff --git a/src/rezplugins/build_system/custom.py b/src/rezplugins/build_system/custom.py index 3b7a5331f..fea0034b6 100644 --- a/src/rezplugins/build_system/custom.py +++ b/src/rezplugins/build_system/custom.py @@ -13,8 +13,8 @@ import os from rez.build_system import BuildSystem -from rez.build_process_ import BuildType -from rez.packages_ import get_developer_package +from rez.build_process import BuildType +from rez.packages import get_developer_package from rez.resolved_context import ResolvedContext from rez.exceptions import PackageMetadataError from rez.utils.colorize import heading, Printer diff --git a/src/rezplugins/package_repository/filesystem.py b/src/rezplugins/package_repository/filesystem.py index 5c63cde55..8edc49067 100644 --- a/src/rezplugins/package_repository/filesystem.py +++ b/src/rezplugins/package_repository/filesystem.py @@ -10,7 +10,7 @@ import platform from rez.package_repository import PackageRepository -from rez.package_resources_ import PackageFamilyResource, VariantResourceHelper, \ +from rez.package_resources import PackageFamilyResource, VariantResourceHelper, \ PackageResourceHelper, package_pod_schema, \ package_release_keys, package_build_only_keys from rez.serialise import clear_file_caches, open_file_for_write diff --git a/src/rezplugins/package_repository/memory.py b/src/rezplugins/package_repository/memory.py index 88da55073..080f2748c 100644 --- a/src/rezplugins/package_repository/memory.py +++ b/src/rezplugins/package_repository/memory.py @@ -2,7 +2,7 @@ In-memory package repository """ from rez.package_repository import PackageRepository -from rez.package_resources_ import PackageFamilyResource, PackageResource, \ +from rez.package_resources import PackageFamilyResource, PackageResource, \ VariantResourceHelper, PackageResourceHelper, package_pod_schema from rez.exceptions import PackageMetadataError from rez.utils.formatting import is_valid_package_name, PackageRequest