From 6cfdb7d242efe2926c9003abb8a19f08b1a8e904 Mon Sep 17 00:00:00 2001 From: ajohns Date: Tue, 2 Feb 2021 09:11:51 +1100 Subject: [PATCH 1/8] -context retargeting implemented -relative-pathed fs pkg repo still to come --- src/rez/package_repository.py | 19 ++++++++++ src/rez/resolved_context.py | 66 +++++++++++++++++++++++++++++++++-- src/rez/tests/test_context.py | 32 +++++++++++++---- src/rez/utils/_version.py | 2 +- 4 files changed, 110 insertions(+), 9 deletions(-) diff --git a/src/rez/package_repository.py b/src/rez/package_repository.py index 19b0cee82..6a2f4f25c 100644 --- a/src/rez/package_repository.py +++ b/src/rez/package_repository.py @@ -230,6 +230,25 @@ def install_variant(self, variant_resource, dry_run=False, overrides=None): """ raise NotImplementedError + def get_equivalent_variant(self, variant_resource): + """Find a variant in this repository that is equivalent to that given. + + A variant is equivalent to another if it belongs to a package of the + same name and version, and it has the same definition (ie package + requirements). + + Note that even though the implementation is trivial, this function is + provided since using `install_variant` to find an existing variant is + nonintuitive. + + Args: + variant_resource (`VariantResource`): Variant to install. + + Returns: + `VariantResource` object, or None if the variant was not found. + """ + return self.install_variant(variant_resource, dry_run=True) + def get_parent_package_family(self, package_resource): """Get the parent package family of the given package. diff --git a/src/rez/resolved_context.py b/src/rez/resolved_context.py index a8b8b91ce..5e5e8522b 100644 --- a/src/rez/resolved_context.py +++ b/src/rez/resolved_context.py @@ -27,7 +27,7 @@ from rez.package_cache import PackageCache from rez.shells import create_shell from rez.exceptions import ResolvedContextError, PackageCommandError, \ - RezError, _NeverError, PackageCacheError + RezError, _NeverError, PackageCacheError, PackageNotFoundError from rez.utils.graph_utils import write_dot, write_compacted, read_graph_from_string from rez.vendor.six import six from rez.vendor.version.version import VersionRange @@ -226,7 +226,7 @@ def __init__(self, package_requests, verbosity=0, timestamp=None, self.package_orderers = ( PackageOrderList.singleton if package_orderers is None - else package_orders + else package_orderers ) # settings that affect context execution @@ -439,6 +439,68 @@ def copy(self): import copy return copy.copy(self) + def retargeted(self, package_paths, package_names=None, skip_missing=False): + """Create a retargeted copy of this context. + + Retargeting a context means replacing its variant references with + the same variants from other package repositories. + + Note that `package_paths` can contains relative filepaths for filesystem + repositories. In this case, the context will expect the repo to be + stored relative to `self.load_path`, and will disable memcached when + accessing that repo. This functionality is used by `rez-bundle`. + + Args: + package_paths: List of paths to search for pkgs to retarget to. + package_names (list of str): Only retarget these packages. If None, + retarget all packages. + skip_missing (bool): If True, skip retargeting of variants that + cannot be found in `package_paths`. By default, a + `PackageNotFoundError` is raised. + + Returns: + ResolvecContext`: The retargeted context. + """ + retargeted_variants = [] + + pkg_repos = [ + package_repository_manager.get_repository(x) + for x in package_paths + ] + + # find retargeted variant for every variant in this context + for src_variant in (self._resolved_packages or []): + if package_names is not None and src_variant.name not in package_names: + retargeted_variants.append(src_variant) + continue + + found = None + + for pkg_repo in pkg_repos: + dest_variant = pkg_repo.get_equivalent_variant(src_variant.resource) + if dest_variant is not None: + found = True + break + + if not found: + if skip_missing: + retargeted_variants.append(src_variant) + continue + else: + raise PackageNotFoundError( + "The equivalent variant in package %s could not be found in any of %r" + % (src_variant.parent, package_paths) + ) + + retargeted_variants.append(dest_variant) + + # create the retargeted context + d = self.to_dict() + d["resolved_packages"] = [ + x.handle.to_dict() for x in retargeted_variants + ] + return self.from_dict(d) + # TODO: deprecate in favor of patch() method def get_patched_request(self, package_requests=None, package_subtractions=None, strict=False, rank=0): diff --git a/src/rez/tests/test_context.py b/src/rez/tests/test_context.py index 6bcde329e..4d526cf90 100644 --- a/src/rez/tests/test_context.py +++ b/src/rez/tests/test_context.py @@ -8,6 +8,7 @@ from rez.utils.platform_ import platform_ import unittest import subprocess +import shutil import os.path import os @@ -17,12 +18,12 @@ class TestContext(TestBase, TempdirMixin): def setUpClass(cls): TempdirMixin.setUpClass() - packages_path = os.path.join(cls.root, "packages") - os.makedirs(packages_path) - hello_world.bind(packages_path) + cls.packages_path = os.path.join(cls.root, "packages") + os.makedirs(cls.packages_path) + hello_world.bind(cls.packages_path) cls.settings = dict( - packages_path=[packages_path], + packages_path=[cls.packages_path], package_filter=None, implicit_packages=[], warn_untimestamped=False, @@ -65,15 +66,17 @@ def test_execute_command(self): def test_execute_command_environ(self): """Test that execute_command properly sets environ dict.""" - parent_environ = {"BIGLY": "covfefe"} r = ResolvedContext(["hello_world"]) + self._test_execute_command_environ(r) + def _test_execute_command_environ(self, r): pycode = ("import os; " "print(os.getenv(\"BIGLY\")); " "print(os.getenv(\"OH_HAI_WORLD\"))") args = ["python", "-c", pycode] + parent_environ = {"BIGLY": "covfefe"} p = r.execute_command(args, parent_environ=parent_environ, stdout=subprocess.PIPE) stdout, _ = p.communicate() @@ -83,7 +86,7 @@ def test_execute_command_environ(self): self.assertEqual(parts, ["covfefe", "hello"]) def test_serialize(self): - """Test context serlialzation.""" + """Test context serialization.""" # save file = os.path.join(self.root, "test.rxt") @@ -98,6 +101,23 @@ def test_serialize(self): env = r2.get_environ() self.assertEqual(env.get("OH_HAI_WORLD"), "hello") + def test_retarget(self): + """Test that a retargeted context behaves identically.""" + + # make a copy of the pkg repo + packages_path2 = os.path.join(self.root, "packages2") + shutil.copytree(self.packages_path, packages_path2) + + # create a context, retarget to pkg repo copy + r = ResolvedContext(["hello_world"]) + r2 = r.retargeted(package_paths=[packages_path2]) + + # check the pkg we contain is in the copied pkg repo + variant = r2.resolved_packages[0] + self.assertTrue(variant.root.startswith(packages_path2 + os.path.sep)) + + self._test_execute_command_environ(r2) + if __name__ == '__main__': unittest.main() diff --git a/src/rez/utils/_version.py b/src/rez/utils/_version.py index 20ce37a51..496a45669 100644 --- a/src/rez/utils/_version.py +++ b/src/rez/utils/_version.py @@ -1,7 +1,7 @@ # Update this value to version up Rez. Do not place anything else in this file. -_rez_version = "2.72.0" +_rez_version = "2.73.0" # Copyright 2013-2016 Allan Johns. From f99572d604f19f689ce3558102d2ab64fda1c33c Mon Sep 17 00:00:00 2001 From: ajohns Date: Tue, 2 Feb 2021 10:10:41 +1100 Subject: [PATCH 2/8] debugging failing test on osx --- src/rez/tests/test_context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rez/tests/test_context.py b/src/rez/tests/test_context.py index 4d526cf90..fd1cd424e 100644 --- a/src/rez/tests/test_context.py +++ b/src/rez/tests/test_context.py @@ -114,7 +114,8 @@ def test_retarget(self): # check the pkg we contain is in the copied pkg repo variant = r2.resolved_packages[0] - self.assertTrue(variant.root.startswith(packages_path2 + os.path.sep)) + prefix = packages_path2 + os.path.sep + self.assertEqual(variant.root[:len(prefix)], prefix) self._test_execute_command_environ(r2) From 373f423c81da9b52a2bbdb458708c00f61f552b8 Mon Sep 17 00:00:00 2001 From: ajohns Date: Tue, 2 Feb 2021 10:25:08 +1100 Subject: [PATCH 3/8] account for fs symlinks in context retarget unit test --- src/rez/tests/test_context.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rez/tests/test_context.py b/src/rez/tests/test_context.py index fd1cd424e..cbf4e26ce 100644 --- a/src/rez/tests/test_context.py +++ b/src/rez/tests/test_context.py @@ -114,8 +114,11 @@ def test_retarget(self): # check the pkg we contain is in the copied pkg repo variant = r2.resolved_packages[0] + packages_path2 = os.path.realpath(packages_path2) + variant_root = os.path.realpath(variant.root) + prefix = packages_path2 + os.path.sep - self.assertEqual(variant.root[:len(prefix)], prefix) + self.assertEqual(variant_root[:len(prefix)], prefix) self._test_execute_command_environ(r2) From 550b303b590776d865ee3200dd6ca1411271201a Mon Sep 17 00:00:00 2001 From: ajohns Date: Tue, 2 Feb 2021 14:02:50 +1100 Subject: [PATCH 4/8] support disabling of memcaching in filesystem pkg repo --- src/rez/package_repository.py | 19 +++++-- .../package_repository/filesystem.py | 56 +++++++++++++------ 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/src/rez/package_repository.py b/src/rez/package_repository.py index 6a2f4f25c..cfeb2e083 100644 --- a/src/rez/package_repository.py +++ b/src/rez/package_repository.py @@ -421,7 +421,7 @@ def __init__(self, resource_pool=None): self.pool = resource_pool self.repositories = {} - def get_repository(self, path): + def get_repository(self, path, **repo_args): """Get a package repository. Args: @@ -429,6 +429,7 @@ def get_repository(self, path): simply be a path (which is managed by the 'filesystem' package repository plugin), or a string in the form "type@location", where 'type' identifies the repository plugin type to use. + repo_args (kwargs): Extra constructor args for the repo. Returns: `PackageRepository` instance. @@ -451,11 +452,17 @@ def get_repository(self, path): normalised_path = "%s@%s" % (repo_type, location) # get possibly cached repo - repository = self.repositories.get(normalised_path) + if repo_args: + key = tuple([normalised_path] + sorted(repo_args.items())) + else: + key = normalised_path + repository = self.repositories.get(key) + + # create and cache if not already cached if repository is None: - repository = self._get_repository(normalised_path) - self.repositories[normalised_path] = repository + repository = self._get_repository(normalised_path, **repo_args) + self.repositories[key] = repository return repository @@ -523,10 +530,10 @@ def clear_caches(self): self.repositories.clear() self.pool.clear_caches() - def _get_repository(self, path): + def _get_repository(self, path, **repo_args): repo_type, location = path.split('@', 1) cls = plugin_manager.get_plugin_class('package_repository', repo_type) - repo = cls(location, self.pool) + repo = cls(location, self.pool, **repo_args) return repo diff --git a/src/rezplugins/package_repository/filesystem.py b/src/rezplugins/package_repository/filesystem.py index b970fb487..24086c446 100644 --- a/src/rezplugins/package_repository/filesystem.py +++ b/src/rezplugins/package_repository/filesystem.py @@ -7,13 +7,13 @@ import stat import errno import time -import platform from rez.package_repository import PackageRepository from rez.package_resources import PackageFamilyResource, VariantResourceHelper, \ PackageResourceHelper, package_pod_schema, \ package_release_keys, package_build_only_keys -from rez.serialise import clear_file_caches, open_file_for_write +from rez.serialise import clear_file_caches, open_file_for_write, load_from_file, \ + FileFormat from rez.package_serialise import dump_package_data from rez.exceptions import PackageMetadataError, ResourceError, RezSystemError, \ ConfigurationError, PackageRepositoryError @@ -23,7 +23,6 @@ from rez.utils.memcached import memcached, pool_memcached_connections from rez.utils.filesystem import make_path_writable, canonical_path from rez.utils.platform_ import platform_ -from rez.serialise import load_from_file, FileFormat from rez.config import config from rez.backport.lru_cache import lru_cache from rez.vendor.schema.schema import Schema, Optional, And, Use, Or @@ -186,7 +185,12 @@ def _load(self): raise PackageDefinitionFileMissing( "Missing package definition file: %r" % self) - data = load_from_file(self.filepath, self.file_format) + data = load_from_file( + self.filepath, + self.file_format, + disable_memcache=self._repository.disable_memcache + ) + check_format_version(self.filepath, data) if "timestamp" not in data: # old format support @@ -323,7 +327,12 @@ def iter_packages(self): def _load(self): format_ = FileFormat[self.ext] - data = load_from_file(self.filepath, format_) + data = load_from_file( + self.filepath, + format_, + disable_memcache=self._repository.disable_memcache + ) + check_format_version(self.filepath, data) return data @@ -468,7 +477,7 @@ class FileSystemPackageRepository(PackageRepository): def name(cls): return "filesystem" - def __init__(self, location, resource_pool): + def __init__(self, location, resource_pool, disable_memcache=False): """Create a filesystem package repository. Args: @@ -480,6 +489,7 @@ def __init__(self, location, resource_pool): location = canonical_path(location, platform_) super(FileSystemPackageRepository, self).__init__(location, resource_pool) + self.disable_memcache = disable_memcache global _settings _settings = config.plugins.package_repository.filesystem @@ -498,6 +508,24 @@ def __init__(self, location, resource_pool): self.get_variants = lru_cache(maxsize=None)(self._get_variants) self.get_file = lru_cache(maxsize=None)(self._get_file) + # decorate with memcachemed memoizers unless told otherwise + if not self.disable_memcache: + decorator1 = memcached( + servers=config.memcached_uri if config.cache_listdir else None, + min_compress_len=config.memcached_listdir_min_compress_len, + key=self._get_family_dirs__key, + debug=config.debug_memcache + ) + self._get_family_dirs = decorator1(self._get_family_dirs) + + decorator2 = memcached( + servers=config.memcached_uri if config.cache_listdir else None, + min_compress_len=config.memcached_listdir_min_compress_len, + key=self._get_version_dirs__key, + debug=config.debug_memcache + ) + self._get_version_dirs = decorator2(self._get_version_dirs) + def _uid(self): t = ["filesystem", self.location] if os.path.exists(self.location): @@ -796,8 +824,11 @@ def clear_caches(self): self.get_packages.cache_clear() self.get_variants.cache_clear() self.get_file.cache_clear() - self._get_family_dirs.forget() - self._get_version_dirs.forget() + + if not self.disable_memcache: + self._get_family_dirs.forget() + self._get_version_dirs.forget() + # unfortunately we need to clear file cache across the board clear_file_caches() @@ -818,10 +849,6 @@ def _get_family_dirs__key(self): else: return str(("listdir", self.location)) - @memcached(servers=config.memcached_uri if config.cache_listdir else None, - min_compress_len=config.memcached_listdir_min_compress_len, - key=_get_family_dirs__key, - debug=config.debug_memcache) def _get_family_dirs(self): dirs = [] if not os.path.isdir(self.location): @@ -843,12 +870,7 @@ def _get_version_dirs__key(self, root): st = os.stat(root) return str(("listdir", root, int(st.st_ino), st.st_mtime)) - @memcached(servers=config.memcached_uri if config.cache_listdir else None, - min_compress_len=config.memcached_listdir_min_compress_len, - key=_get_version_dirs__key, - debug=config.debug_memcache) def _get_version_dirs(self, root): - # Ignore a version if there is a .ignore file next to it def ignore_dir(name): path = os.path.join(root, self.ignore_prefix + name) From c0392556b331a68b54bd90633d54659f6cc40ba2 Mon Sep 17 00:00:00 2001 From: ajohns Date: Tue, 16 Feb 2021 20:38:57 +1100 Subject: [PATCH 5/8] -first pass context bundling implemented -added new 'settings.yaml' in root of filesystem pkg repos -only works for 'disable_memcached' currently --- src/rez/bundle_context.py | 130 ++++++++++++++++++ src/rez/cli/_entry_points.py | 7 + src/rez/cli/_main.py | 4 +- src/rez/cli/_util.py | 1 + src/rez/cli/bundle.py | 48 +++++++ src/rez/exceptions.py | 5 + src/rez/package_repository.py | 14 +- src/rez/resolved_context.py | 102 +++++++++++--- src/rez/tests/test_context.py | 7 +- src/rez/utils/yaml.py | 7 + .../package_repository/filesystem.py | 19 ++- 11 files changed, 308 insertions(+), 36 deletions(-) create mode 100644 src/rez/bundle_context.py create mode 100644 src/rez/cli/bundle.py diff --git a/src/rez/bundle_context.py b/src/rez/bundle_context.py new file mode 100644 index 000000000..bb302fd01 --- /dev/null +++ b/src/rez/bundle_context.py @@ -0,0 +1,130 @@ +import os +import os.path + +from rez.exceptions import ContextBundleError +from rez.utils.logging_ import print_info, print_warning +from rez.utils.yaml import save_yaml +from rez.package_copy import copy_package + + +def bundle_context(context, dest_dir, force=False, skip_non_relocatable=False, + quiet=False, verbose=False): + """Bundle a context and its variants into a relocatable dir. + + This creates a copy of a context with its variants retargeted to a local + package repository containing only the variants the context uses. The + generated file structure looks like so: + + /dest_dir/ + /context.rxt + /packages/ + /foo/1.1.1/package.py + /...(payload)... + /bah/4.5.6/package.py + /...(payload)... + + Args: + context (`ResolvedContext`): Context to bundle + dest_dir (str): Destination directory. Must not exist. + force (bool): If True, relocate package even if non-relocatable. Use at + your own risk. Overrides `skip_non_relocatable`. + skip_non_relocatable (bool): If True, leave non-relocatable packages + unchanged. Normally this will raise a `PackageCopyError`. + quiet (bool): Suppress all output + verbose (bool): Verbose mode (quiet will override) + """ + if quiet: + verbose = False + if force: + skip_non_relocatable = False + + if os.path.exists(dest_dir): + raise ContextBundleError("Dest dir must not exist: %s" % dest_dir) + + if not quiet: + label = context.load_path or "context" + print_info("Bundling %s into %s...", label, dest_dir) + + os.mkdir(dest_dir) + + _init_bundle(dest_dir) + + relocated_package_names = _copy_variants( + context=context, + bundle_dir=dest_dir, + force=force, + skip_non_relocatable=skip_non_relocatable, + verbose=verbose + ) + + rxt_filepath = _write_retargeted_context( + context=context, + bundle_dir=dest_dir, + relocated_package_names=relocated_package_names + ) + + if verbose: + print_info("Context bundled to %s", rxt_filepath) + + +def _init_bundle(bundle_dir): + # Create bundle conf file. It doesn't contain anything at time of writing, + # but its presence on disk signifies that this is a context bundle. + # + bundle_filepath = os.path.join(bundle_dir, "bundle.yaml") + save_yaml(bundle_filepath) + + # init pkg repo + repo_path = os.path.join(bundle_dir, "packages") + os.mkdir(repo_path) + + # Bundled repos are always memcached disabled because they're on local disk + # (so access should be fast); but also, local repo paths written to shared + # memcached instance could easily clash. + # + settings_filepath = os.path.join(bundle_dir, "packages", "settings.yaml") + save_yaml(settings_filepath, disable_memcached=True) + + +def _copy_variants(context, bundle_dir, force=False, skip_non_relocatable=False, + verbose=False): + relocated_package_names = [] + repo_path = os.path.join(bundle_dir, "packages") + + for variant in context.resolved_packages: + package = variant.parent + + if skip_non_relocatable and not package.is_relocatable: + if verbose: + print_warning( + "Skipped bundling of non-relocatable package %s", + package.qualified_name + ) + continue + + copy_package( + package=package, + dest_repository=repo_path, + variants=[variant.index], + force=force, + keep_timestamp=True, + verbose=verbose + ) + + relocated_package_names.append(package.name) + + return relocated_package_names + + +def _write_retargeted_context(context, bundle_dir, relocated_package_names): + repo_path = os.path.join(bundle_dir, "packages") + rxt_filepath = os.path.join(bundle_dir, "context.rxt") + + bundled_context = context.retargeted( + package_paths=[repo_path], + package_names=relocated_package_names, + skip_missing=True + ) + + bundled_context.save(rxt_filepath) + return rxt_filepath diff --git a/src/rez/cli/_entry_points.py b/src/rez/cli/_entry_points.py index 1f8023bde..7a94f78ee 100644 --- a/src/rez/cli/_entry_points.py +++ b/src/rez/cli/_entry_points.py @@ -253,3 +253,10 @@ def run_rez_yaml2py(): check_production_install() from rez.cli._main import run return run("yaml2py") + + +@scriptname("rez-bundle") +def run_rez_bundle(): + check_production_install() + from rez.cli._main import run + return run("bundle") diff --git a/src/rez/cli/_main.py b/src/rez/cli/_main.py index ea1031cc7..ec13215fe 100644 --- a/src/rez/cli/_main.py +++ b/src/rez/cli/_main.py @@ -8,7 +8,7 @@ from argparse import _StoreTrueAction, SUPPRESS from rez.cli._util import subcommands, LazyArgumentParser, _env_var_true from rez.utils.logging_ import print_error -from rez.exceptions import RezError, RezSystemError +from rez.exceptions import RezError, RezSystemError, _NeverError from rez import __version__ @@ -146,7 +146,7 @@ def run(command=None): extra_arg_groups = [] if opts.debug or _env_var_true("REZ_DEBUG"): - exc_type = None + exc_type = _NeverError else: exc_type = RezError diff --git a/src/rez/cli/_util.py b/src/rez/cli/_util.py index b3b741e4b..ceac613e7 100644 --- a/src/rez/cli/_util.py +++ b/src/rez/cli/_util.py @@ -56,6 +56,7 @@ "test": {}, "view": {}, "yaml2py": {}, + "bundle": {} } diff --git a/src/rez/cli/bundle.py b/src/rez/cli/bundle.py new file mode 100644 index 000000000..a48adf49c --- /dev/null +++ b/src/rez/cli/bundle.py @@ -0,0 +1,48 @@ +''' +Bundle a context and its packages into a relocatable dir. +''' +from __future__ import print_function + +import os +import os.path +import sys + + +def setup_parser(parser, completions=False): + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-s", "--skip-non-relocatable", action="store_true", + help="leave non-relocatable packages non-bundled, rather than raise an error") + group.add_argument( + "-f", "--force", action="store_true", + help="bundle package even if it isn't relocatable (use at your own risk)") + parser.add_argument( + "RXT", + help="context to bundle") + parser.add_argument( + "DEST_DIR", + help="directory to create bundle in; must not exist") + + +def command(opts, parser, extra_arg_groups=None): + from rez.utils.logging_ import print_error + from rez.bundle_context import bundle_context + from rez.resolved_context import ResolvedContext + + rxt_filepath = os.path.abspath(os.path.expanduser(opts.RXT)) + dest_dir = os.path.abspath(os.path.expanduser(opts.DEST_DIR)) + + # sanity checks + if not os.path.exists(rxt_filepath): + print_error("File does not exist: %s", rxt_filepath) + sys.exit(1) + + context = ResolvedContext.load(rxt_filepath) + + bundle_context( + context=context, + dest_dir=dest_dir, + force=opts.force, + skip_non_relocatable=opts.skip_non_relocatable, + verbose=opts.verbose + ) diff --git a/src/rez/exceptions.py b/src/rez/exceptions.py index a70415aec..1b9b9f5b9 100644 --- a/src/rez/exceptions.py +++ b/src/rez/exceptions.py @@ -93,6 +93,11 @@ class PackageCopyError(RezError): pass +class ContextBundleError(RezError): + """There was a problem bundling a context.""" + pass + + class PackageCacheError(RezError): """There was an error related to a package cache.""" pass diff --git a/src/rez/package_repository.py b/src/rez/package_repository.py index cfeb2e083..53faa0367 100644 --- a/src/rez/package_repository.py +++ b/src/rez/package_repository.py @@ -421,7 +421,7 @@ def __init__(self, resource_pool=None): self.pool = resource_pool self.repositories = {} - def get_repository(self, path, **repo_args): + def get_repository(self, path): """Get a package repository. Args: @@ -429,7 +429,6 @@ def get_repository(self, path, **repo_args): simply be a path (which is managed by the 'filesystem' package repository plugin), or a string in the form "type@location", where 'type' identifies the repository plugin type to use. - repo_args (kwargs): Extra constructor args for the repo. Returns: `PackageRepository` instance. @@ -452,17 +451,12 @@ def get_repository(self, path, **repo_args): normalised_path = "%s@%s" % (repo_type, location) # get possibly cached repo - if repo_args: - key = tuple([normalised_path] + sorted(repo_args.items())) - else: - key = normalised_path - - repository = self.repositories.get(key) + repository = self.repositories.get(normalised_path) # create and cache if not already cached if repository is None: - repository = self._get_repository(normalised_path, **repo_args) - self.repositories[key] = repository + repository = self._get_repository(normalised_path) + self.repositories[normalised_path] = repository return repository diff --git a/src/rez/resolved_context.py b/src/rez/resolved_context.py index 5e5e8522b..877b08b03 100644 --- a/src/rez/resolved_context.py +++ b/src/rez/resolved_context.py @@ -13,7 +13,7 @@ from rez.utils.formatting import columnise, PackageRequest, ENV_VAR_REGEX, \ header_comment, minor_header_comment from rez.utils.data_utils import deep_del -from rez.utils.filesystem import TempDirs +from rez.utils.filesystem import TempDirs, is_subdirectory from rez.utils.memcached import pool_memcached_connections from rez.utils.logging_ import print_error, print_warning from rez.backport.shutilwhich import which @@ -37,6 +37,7 @@ from rez.utils import json from rez.utils.yaml import dump_yaml +from contextlib import contextmanager from functools import wraps import getpass import socket @@ -123,13 +124,12 @@ class ResolvedContext(object): command within a configured python namespace, without spawning a child shell. """ - serialize_version = (4, 6) + serialize_version = (4, 7) tmpdir_manager = TempDirs(config.context_tmpdir, prefix="rez_context_") - context_tracking_payload = None context_tracking_lock = threading.Lock() - package_cache_present = True + local = threading.local() class Callback(object): def __init__(self, max_fails, time_limit, callback, buf=None): @@ -445,11 +445,6 @@ def retargeted(self, package_paths, package_names=None, skip_missing=False): Retargeting a context means replacing its variant references with the same variants from other package repositories. - Note that `package_paths` can contains relative filepaths for filesystem - repositories. In this case, the context will expect the repo to be - stored relative to `self.load_path`, and will disable memcached when - accessing that repo. This functionality is used by `rez-bundle`. - Args: package_paths: List of paths to search for pkgs to retarget to. package_names (list of str): Only retarget these packages. If None, @@ -496,9 +491,14 @@ def retargeted(self, package_paths, package_names=None, skip_missing=False): # create the retargeted context d = self.to_dict() - d["resolved_packages"] = [ - x.handle.to_dict() for x in retargeted_variants - ] + + d.update({ + "package_paths": package_paths, + "resolved_packages": [ + x.handle.to_dict() for x in retargeted_variants + ] + }) + return self.from_dict(d) # TODO: deprecate in favor of patch() method @@ -635,8 +635,9 @@ def graph(self, as_dot=False): def save(self, path): """Save the resolved context to file.""" - with open(path, 'w') as f: - self.write_to_buffer(f) + with self._detect_bundle(path): + with open(path, 'w') as f: + self.write_to_buffer(f) def write_to_buffer(self, buf): """Save the context to a buffer.""" @@ -680,8 +681,10 @@ def is_current(self): @classmethod def load(cls, path): """Load a resolved context from file.""" - with open(path) as f: - context = cls.read_from_buffer(f, path) + with cls._detect_bundle(path): + with open(path) as f: + context = cls.read_from_buffer(f, path) + context.set_load_path(path) return context @@ -1474,6 +1477,10 @@ def _add(field): resolved_packages.append(pkg.handle.to_dict()) data["resolved_packages"] = resolved_packages + # since serialization version 4.7 + for handle in data["resolved_packages"]: + self._adjust_variant_for_bundling(handle, out=True) + if _add("resolved_ephemerals"): resolved_ephemerals = [] for ephemeral in (self._resolved_ephemerals or []): @@ -1613,6 +1620,9 @@ def _print_version(value): from rez.utils.backcompat import convert_old_variant_handle variant_handle = convert_old_variant_handle(variant_handle) + # -- SINCE SERIALIZE VERSION 4.7 + cls._adjust_variant_for_bundling(variant_handle, out=False) + variant = get_variant(variant_handle) variant.set_context(r) r._resolved_packages.append(variant) @@ -1682,6 +1692,66 @@ def _print_version(value): return r + @classmethod + @contextmanager + def _detect_bundle(cls, path): + bundle_path = None + base_dir = os.path.dirname(os.path.abspath(path)) + bundle_filepath = os.path.join(base_dir, "bundle.yaml") + + try: + if os.path.exists(bundle_filepath): + bundle_path = base_dir + except IOError: + pass + + try: + if bundle_path: + cls.local.bundle_path = bundle_path + + yield + finally: + try: + delattr(cls.local, "bundle_path") + except AttributeError: + pass + + @classmethod + def _get_bundle_path(cls): + return getattr(cls.local, "bundle_path", None) + + @classmethod + def _adjust_variant_for_bundling(cls, handle, out): + """ + Deals with making variant pkg repo ref relative/nonrelative to take + bundling into account. + + Note: Alters `handle` in-place. + """ + bundle_path = cls._get_bundle_path() + if not bundle_path: + return + + vars_ = handle.get("variables", {}) + if vars_.get("repository_type") != "filesystem": + return + + repo_path = vars_["location"] + + # serializing out, make repo relative + if out: + assert os.path.isabs(repo_path) + + if is_subdirectory(repo_path, bundle_path): + vars_["location"] = os.path.relpath(repo_path, bundle_path) + + # serializing in, make repo absolute + else: + if os.path.isabs(repo_path): + return + + vars_["location"] = os.path.join(bundle_path, repo_path) + @classmethod def _get_package_cache(cls): if not cls.package_cache_present: diff --git a/src/rez/tests/test_context.py b/src/rez/tests/test_context.py index cbf4e26ce..5873a6ce2 100644 --- a/src/rez/tests/test_context.py +++ b/src/rez/tests/test_context.py @@ -6,6 +6,7 @@ from rez.resolved_context import ResolvedContext from rez.bind import hello_world from rez.utils.platform_ import platform_ +from rez.utils.filesystem import is_subdirectory import unittest import subprocess import shutil @@ -114,11 +115,7 @@ def test_retarget(self): # check the pkg we contain is in the copied pkg repo variant = r2.resolved_packages[0] - packages_path2 = os.path.realpath(packages_path2) - variant_root = os.path.realpath(variant.root) - - prefix = packages_path2 + os.path.sep - self.assertEqual(variant_root[:len(prefix)], prefix) + self.assertTrue(is_subdirectory(variant.root, packages_path2)) self._test_execute_command_environ(r2) diff --git a/src/rez/utils/yaml.py b/src/rez/utils/yaml.py index 886262ac1..85e650cdf 100644 --- a/src/rez/utils/yaml.py +++ b/src/rez/utils/yaml.py @@ -53,6 +53,13 @@ def load_yaml(filepath): return yaml.load(txt, Loader=yaml.FullLoader) +def save_yaml(filepath, **fields): + """Convenience function for writing yaml-encoded data to disk.""" + content = dump_yaml(fields) + with open(filepath, 'w') as f: + f.write(content + '\n') + + # Copyright 2013-2016 Allan Johns. # # This library is free software: you can redistribute it and/or diff --git a/src/rezplugins/package_repository/filesystem.py b/src/rezplugins/package_repository/filesystem.py index 24086c446..0513d57f4 100644 --- a/src/rezplugins/package_repository/filesystem.py +++ b/src/rezplugins/package_repository/filesystem.py @@ -23,6 +23,7 @@ from rez.utils.memcached import memcached, pool_memcached_connections from rez.utils.filesystem import make_path_writable, canonical_path from rez.utils.platform_ import platform_ +from rez.utils.yaml import load_yaml from rez.config import config from rez.backport.lru_cache import lru_cache from rez.vendor.schema.schema import Schema, Optional, And, Use, Or @@ -477,7 +478,7 @@ class FileSystemPackageRepository(PackageRepository): def name(cls): return "filesystem" - def __init__(self, location, resource_pool, disable_memcache=False): + def __init__(self, location, resource_pool): """Create a filesystem package repository. Args: @@ -489,8 +490,16 @@ def __init__(self, location, resource_pool, disable_memcache=False): location = canonical_path(location, platform_) super(FileSystemPackageRepository, self).__init__(location, resource_pool) - self.disable_memcache = disable_memcache + # load settings optionally defined in a settings.yaml + local_settings = {} + settings_filepath = os.path.join(location, "settings.yaml") + if os.path.exists(settings_filepath): + local_settings.update(load_yaml(settings_filepath)) + + self.disable_memcache = local_settings.get("disable_memcache", False) + + # TODO allow these settings to be overridden in settings.yaml also global _settings _settings = config.plugins.package_repository.filesystem @@ -856,8 +865,12 @@ def _get_family_dirs(self): for name in os.listdir(self.location): path = os.path.join(self.location, name) + + if name in ("settings.yaml", self.file_lock_dir): + continue # skip reserved file/dirnames + if os.path.isdir(path): - if is_valid_package_name(name) and name != self.file_lock_dir: + if is_valid_package_name(name): dirs.append((name, None)) else: name_, ext_ = os.path.splitext(name) From 2d69f7576468d1ce1832fd4f953b5b3d7573b358 Mon Sep 17 00:00:00 2001 From: ajohns Date: Tue, 23 Feb 2021 08:28:31 +1100 Subject: [PATCH 6/8] added bundled context test --- src/rez/tests/test_context.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/rez/tests/test_context.py b/src/rez/tests/test_context.py index 5873a6ce2..08c2477ba 100644 --- a/src/rez/tests/test_context.py +++ b/src/rez/tests/test_context.py @@ -4,6 +4,7 @@ from rez.tests.util import restore_os_environ, restore_sys_path, TempdirMixin, \ TestBase from rez.resolved_context import ResolvedContext +from rez.bundle_context import bundle_context from rez.bind import hello_world from rez.utils.platform_ import platform_ from rez.utils.filesystem import is_subdirectory @@ -119,6 +120,37 @@ def test_retarget(self): self._test_execute_command_environ(r2) + def test_bundled(self): + """Test that a bundled context behaves identically.""" + bundle_path = os.path.join(self.root, "bundle") + + # create context and bundle it + r = ResolvedContext(["hello_world"]) + bundle_context( + context=r, + dest_dir=bundle_path, + force=True, + verbose=True + ) + + def _test_bundle(path): + # load the bundled context + r2 = ResolvedContext.load(os.path.join(path, "context.rxt")) + + # check the pkg we contain is in the bundled pkg repo + variant = r2.resolved_packages[0] + self.assertTrue(is_subdirectory(variant.root, path)) + + self._test_execute_command_environ(r2) + + # test the bundle + _test_bundle(bundle_path) + + # copy the bundle and test the copy + bundle_path2 = os.path.join(self.root, "bundle2") + shutil.copytree(bundle_path, bundle_path2) + _test_bundle(bundle_path2) + if __name__ == '__main__': unittest.main() From 90170b08820d020873083a79bf7d573c12ba3a6d Mon Sep 17 00:00:00 2001 From: ajohns Date: Tue, 23 Feb 2021 10:18:17 +1100 Subject: [PATCH 7/8] fix for resource mismatch when symlinks present in bundle path --- src/rez/resolved_context.py | 15 +++++++++--- src/rez/tests/test_context.py | 43 +++++++++++++++++++++++++++-------- wiki/pages/Bundles.md | 2 ++ wiki/pages/__Sidebar.md | 1 + 4 files changed, 48 insertions(+), 13 deletions(-) create mode 100644 wiki/pages/Bundles.md diff --git a/src/rez/resolved_context.py b/src/rez/resolved_context.py index 877b08b03..8a5d271e0 100644 --- a/src/rez/resolved_context.py +++ b/src/rez/resolved_context.py @@ -13,7 +13,7 @@ from rez.utils.formatting import columnise, PackageRequest, ENV_VAR_REGEX, \ header_comment, minor_header_comment from rez.utils.data_utils import deep_del -from rez.utils.filesystem import TempDirs, is_subdirectory +from rez.utils.filesystem import TempDirs, is_subdirectory, canonical_path from rez.utils.memcached import pool_memcached_connections from rez.utils.logging_ import print_error, print_warning from rez.backport.shutilwhich import which @@ -36,6 +36,7 @@ from rez.vendor import yaml from rez.utils import json from rez.utils.yaml import dump_yaml +from rez.utils.platform_ import platform_ from contextlib import contextmanager from functools import wraps @@ -1743,14 +1744,22 @@ def _adjust_variant_for_bundling(cls, handle, out): assert os.path.isabs(repo_path) if is_subdirectory(repo_path, bundle_path): - vars_["location"] = os.path.relpath(repo_path, bundle_path) + vars_["location"] = os.path.relpath( + os.path.realpath(repo_path), + os.path.realpath(bundle_path) + ) # serializing in, make repo absolute else: if os.path.isabs(repo_path): return - vars_["location"] = os.path.join(bundle_path, repo_path) + # Must make canonical otherwise a symlinked path will cause it not + # to match the repo location, which is always canonical. + # + location = os.path.join(bundle_path, repo_path) + location = canonical_path(location, platform_) + vars_["location"] = location @classmethod def _get_package_cache(cls): diff --git a/src/rez/tests/test_context.py b/src/rez/tests/test_context.py index 08c2477ba..a4898e75f 100644 --- a/src/rez/tests/test_context.py +++ b/src/rez/tests/test_context.py @@ -10,6 +10,7 @@ from rez.utils.filesystem import is_subdirectory import unittest import subprocess +import platform import shutil import os.path import os @@ -122,16 +123,6 @@ def test_retarget(self): def test_bundled(self): """Test that a bundled context behaves identically.""" - bundle_path = os.path.join(self.root, "bundle") - - # create context and bundle it - r = ResolvedContext(["hello_world"]) - bundle_context( - context=r, - dest_dir=bundle_path, - force=True, - verbose=True - ) def _test_bundle(path): # load the bundled context @@ -143,6 +134,17 @@ def _test_bundle(path): self._test_execute_command_environ(r2) + bundle_path = os.path.join(self.root, "bundle") + + # create context and bundle it + r = ResolvedContext(["hello_world"]) + bundle_context( + context=r, + dest_dir=bundle_path, + force=True, + verbose=True + ) + # test the bundle _test_bundle(bundle_path) @@ -151,6 +153,27 @@ def _test_bundle(path): shutil.copytree(bundle_path, bundle_path2) _test_bundle(bundle_path2) + # Create a bundle in a symlinked dest path. Bugs can arise where the + # real path is used in some places and not others. + # + if platform.system().lower() in ("linux", "darwin"): + hard_path = os.path.join(self.root, "foo") + bundles_path = os.path.join(self.root, "bundles") + bundle_path3 = os.path.join(bundles_path, "bundle3") + + os.mkdir(hard_path) + os.symlink(hard_path, bundles_path) + + r = ResolvedContext(["hello_world"]) + bundle_context( + context=r, + dest_dir=bundle_path3, + force=True, + verbose=True + ) + + _test_bundle(bundle_path3) + if __name__ == '__main__': unittest.main() diff --git a/wiki/pages/Bundles.md b/wiki/pages/Bundles.md new file mode 100644 index 000000000..0366ab1b1 --- /dev/null +++ b/wiki/pages/Bundles.md @@ -0,0 +1,2 @@ +## Overview + diff --git a/wiki/pages/__Sidebar.md b/wiki/pages/__Sidebar.md index 61502cd84..5b1ddfc42 100644 --- a/wiki/pages/__Sidebar.md +++ b/wiki/pages/__Sidebar.md @@ -19,6 +19,7 @@ :rocket: rez: - [[Contexts]] +- [[Bundles]] - [[Suites]] - [[Building Packages]] - [[Package Caching]] From b3c5d8b195839c2eaef81bf866935058564ef807 Mon Sep 17 00:00:00 2001 From: ajohns Date: Tue, 23 Feb 2021 14:37:50 +1100 Subject: [PATCH 8/8] docs update --- wiki/pages/Bundles.md | 42 +++++++++++++++++++++++ wiki/pages/Copying-Packages.md | 62 ++++++++++++++++++++++++++++++++++ wiki/pages/Package-Caching.md | 2 +- wiki/pages/__Sidebar.md | 1 + 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 wiki/pages/Copying-Packages.md diff --git a/wiki/pages/Bundles.md b/wiki/pages/Bundles.md index 0366ab1b1..3dafc18e3 100644 --- a/wiki/pages/Bundles.md +++ b/wiki/pages/Bundles.md @@ -1,2 +1,44 @@ ## Overview +A "context bundle" is a directory containing a context (an rxt file), and a +package repository. All packages in the context are stored in the repository, +making the bundle relocatable and standalone. You can copy a bundle onto a +server for example, or into a container, and there are no external references +to shared package repositories. This is in contrast to a typical context, which +contains absolute references to one or more package repositories that are +typically on shared disk storage. + +To create a bundle via command line: + +``` +]$ rez-env foo -o foo.rxt +]$ rez-bundle foo.rxt ./mybundle + +# example of running a command from the bundled context +]$ rez-env -i ./mybundle/context.rxt -- foo-tool +``` + +To create a bundle via API: + +``` +>>> from rez.bundle_context import bundle_context +>>> from rez.resolved_context import ResolvedContext +>>> +>>> c = ResolvedContext(["python-3+", "foo-1.2+<2"]) +>>> bundle_context(c, "./mybundle") +``` + + +## Structure + +A bundle directory looks like this: + +``` +.../mybundle/ + ./context.rxt + ./packages/ + +``` + +Package references in the rxt file are relative (unlike in a standard context, +where they're absolute), and this makes the bundle relocatable. diff --git a/wiki/pages/Copying-Packages.md b/wiki/pages/Copying-Packages.md new file mode 100644 index 000000000..2f670ab40 --- /dev/null +++ b/wiki/pages/Copying-Packages.md @@ -0,0 +1,62 @@ +## Overview + +Packages can be copied from one [package repository](Basic-Concepts#package-repositories) +to another, like so: + +Via commandline: + +``` +]$ rez-cp --dest-path /svr/packages2 my_pkg-1.2.3 +``` + +Via API: + +``` +>>> from rez.package_copy import copy_package +>>> from rez.packages import get_latest_package +>>> +>>> p = get_latest_package("python") +>>> p +Package(FileSystemPackageResource({'location': '/home/ajohns/packages', 'name': 'python', 'repository_type': 'filesystem', 'version': '3.7.4'})) + +>>> r = copy_package(p, "./repo2") +>>> +>>> print(pprint.pformat(r)) +{ + 'copied': [ + ( + Variant(FileSystemVariantResource({'location': '/home/ajohns/packages', 'name': 'python', 'repository_type': 'filesystem', 'index': 0, 'version': '3.7.4'})), + FileSystemVariantResource({'location': '/home/ajohns/repo2', 'name': 'python', 'repository_type': 'filesystem', 'index': 0, 'version': '3.7.4'}) + ) + ], + 'skipped': [] +} +``` + +Copying packages is actually done one variant at a time, and you can copy some +variants of a package if you want, rather than the entire package. The API call's +return value shows what variants were copied - The 2-tuple in `copied` lists the +source (the variant that was copied from) and destination (the variant that was +created) respectively. + +> [[media/icons/warning.png]] Do not simply copy package directories on disk - +> you should always use `rez-cp`. Copying directly on disk is bypassing rez and +> this can cause problems such as a stale resolve cache. Using `rez-cp` gives +> you more control anyway. + +## Enabling + +Copying packages is enabled by default, however you're also able to specify which +packages are and are not _relocatable_, for much the same reasons as given +[here](Package-Caching#enabling). + +You can mark a package as non-relocatable by setting `relocatable = False` in its +package definition file. There are also config settings that affect relocatability +in the event that relocatable is not defined in a package's definition. For example, +see [default_relocatable](Configuring-Rez#default_relocatable), +[default_relocatable_per_package](Configuring-Rez#default_relocatable_per_package) +and [default_relocatable_per_repository](Configuring-Rez#default_relocatable_per_repository). + +Attempting to copy a non-relocatable package will raise a `PackageCopyError`. +However, note that there is a `force` option that will override this - use at +your own risk. diff --git a/wiki/pages/Package-Caching.md b/wiki/pages/Package-Caching.md index ec3553026..304e7ec4f 100644 --- a/wiki/pages/Package-Caching.md +++ b/wiki/pages/Package-Caching.md @@ -23,7 +23,7 @@ linked to them in a way that doesn't support library relocation. There are also config settings that affect cachability in the event that `cachable` is not defined in a package's definition. For example, see -[default_cachable](Configuring-Rez#), +[default_cachable](Configuring-Rez#default_cachable), [default_cachable_per_package](Configuring-Rez#default_cachable_per_package) and [default_cachable_per_repository](Configuring-Rez#default_cachable_per_repository). diff --git a/wiki/pages/__Sidebar.md b/wiki/pages/__Sidebar.md index 5b1ddfc42..d71540dbe 100644 --- a/wiki/pages/__Sidebar.md +++ b/wiki/pages/__Sidebar.md @@ -22,6 +22,7 @@ - [[Bundles]] - [[Suites]] - [[Building Packages]] +- [[Copying Packages]] - [[Package Caching]] - [[Environment Variables]] - [[Command Line Tools]]