From eaf7696ffed3de0d3994cd01544fe3dd9fa6a44c Mon Sep 17 00:00:00 2001 From: James Date: Tue, 24 Sep 2024 15:11:39 +0200 Subject: [PATCH] Feature/build compatibles (#16871) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * workin in --build=compatible * wip * wip * fix test * wip * new test and --build=compatible syntax * Update conans/client/graph/graph_binaries.py Co-authored-by: Abril Rincón Blanco * allowing ``conan config install-pkg --url`` for initial config (#16876) * allowing ``conan config install-pkg --url`` for initial config * review * revert Meson system=baremetal (#16929) * improve error messages for broken 'list --graph' arguments * minor test refactors * add message for build compatibles * Update test/integration/package_id/compatible_test.py Co-authored-by: Francisco Ramírez * Update test/integration/package_id/compatible_test.py --------- Co-authored-by: Abril Rincón Blanco Co-authored-by: Francisco Ramírez --- conans/client/graph/build_mode.py | 23 +++ conans/client/graph/compute_pid.py | 18 +- conans/client/graph/graph_binaries.py | 33 +++- .../integration/package_id/compatible_test.py | 170 ++++++++++++++++++ 4 files changed, 230 insertions(+), 14 deletions(-) diff --git a/conans/client/graph/build_mode.py b/conans/client/graph/build_mode.py index f7a3cd61f81..8efbd99da3b 100644 --- a/conans/client/graph/build_mode.py +++ b/conans/client/graph/build_mode.py @@ -16,6 +16,8 @@ def __init__(self, params): self.patterns = [] self.build_missing_patterns = [] self._build_missing_excluded = [] + self._build_compatible_patterns = [] + self._build_compatible_excluded = [] self._excluded_patterns = [] if params is None: return @@ -39,6 +41,14 @@ def __init__(self, params): self._build_missing_excluded.append(clean_pattern[1:]) else: self.build_missing_patterns.append(clean_pattern) + elif param == "compatible": + self._build_compatible_patterns = ["*"] + elif param.startswith("compatible:"): + clean_pattern = param[len("compatible:"):] + if clean_pattern and clean_pattern[0] in ["!", "~"]: + self._build_compatible_excluded.append(clean_pattern[1:]) + else: + self._build_compatible_patterns.append(clean_pattern) else: clean_pattern = param if clean_pattern and clean_pattern[0] in ["!", "~"]: @@ -87,8 +97,21 @@ def allowed(self, conan_file): return True if self.should_build_missing(conan_file): return True + if self.allowed_compatible(conan_file): + return True return False + def allowed_compatible(self, conanfile): + if self._build_compatible_excluded: + for pattern in self._build_compatible_excluded: + if ref_matches(conanfile.ref, pattern, is_consumer=False): + return False + return True # If it has not been excluded by the negated patterns, it is included + + for pattern in self._build_compatible_patterns: + if ref_matches(conanfile.ref, pattern, is_consumer=conanfile._conan_is_consumer): + return True + def should_build_missing(self, conanfile): if self._build_missing_excluded: for pattern in self._build_missing_excluded: diff --git a/conans/client/graph/compute_pid.py b/conans/client/graph/compute_pid.py index 9927e657ecd..4ba62318550 100644 --- a/conans/client/graph/compute_pid.py +++ b/conans/client/graph/compute_pid.py @@ -50,15 +50,6 @@ def compute_package_id(node, modes, config_version): config_version=config_version.copy() if config_version else None) conanfile.original_info = conanfile.info.clone() - if hasattr(conanfile, "validate_build"): - with conanfile_exception_formatter(conanfile, "validate_build"): - with conanfile_remove_attr(conanfile, ['cpp_info'], "validate_build"): - try: - conanfile.validate_build() - except ConanInvalidConfiguration as e: - # This 'cant_build' will be ignored if we don't have to build the node. - conanfile.info.cant_build = str(e) - run_validate_package_id(conanfile) if conanfile.info.settings_target: @@ -71,6 +62,15 @@ def compute_package_id(node, modes, config_version): def run_validate_package_id(conanfile): # IMPORTANT: This validation code must run before calling info.package_id(), to mark "invalid" + if hasattr(conanfile, "validate_build"): + with conanfile_exception_formatter(conanfile, "validate_build"): + with conanfile_remove_attr(conanfile, ['cpp_info'], "validate_build"): + try: + conanfile.validate_build() + except ConanInvalidConfiguration as e: + # This 'cant_build' will be ignored if we don't have to build the node. + conanfile.info.cant_build = str(e) + if hasattr(conanfile, "validate"): with conanfile_exception_formatter(conanfile, "validate"): with conanfile_remove_attr(conanfile, ['cpp_info'], "validate"): diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index e5c11174535..e3c6483aebf 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -21,7 +21,7 @@ from conans.util.files import load -class GraphBinariesAnalyzer(object): +class GraphBinariesAnalyzer: def __init__(self, conan_app, global_conf): self._cache = conan_app.cache @@ -152,7 +152,7 @@ def _find_existing_compatible_binaries(self, node, compatibles, remotes, update) conanfile = node.conanfile original_binary = node.binary original_package_id = node.package_id - + conanfile.output.info(f"Main binary package '{original_package_id}' missing") conanfile.output.info(f"Checking {len(compatibles)} compatible configurations") for package_id, compatible_package in compatibles.items(): if should_update_reference(node.ref, update): @@ -180,24 +180,47 @@ def _find_existing_compatible_binaries(self, node, compatibles, remotes, update) node.binary = original_binary node._package_id = original_package_id + def _find_build_compatible_binary(self, node, compatibles): + original_binary = node.binary + original_package_id = node.package_id + output = node.conanfile.output + output.info(f"Requested binary package '{original_package_id}' invalid, can't be built") + output.info(f"Checking {len(compatibles)} configurations, to build a compatible one, " + f"as requested by '--build=compatible'") + for pkg_id, compatible in compatibles.items(): + if not compatible.cant_build: + node._package_id = pkg_id # Modifying package id under the hood, FIXME + self._compatible_found(node.conanfile, pkg_id, compatible) + node.binary = BINARY_BUILD + return + node.binary = original_binary + node._package_id = original_package_id + def _evaluate_node(self, node, build_mode, remotes, update): assert node.binary is None, "Node.binary should be None" assert node.package_id is not None, "Node.package_id shouldn't be None" assert node.prev is None, "Node.prev should be None" self._process_node(node, build_mode, remotes, update) - original_package_id = node.package_id + compatibles = None + if node.binary == BINARY_MISSING \ and not build_mode.should_build_missing(node.conanfile) and not node.should_build: compatibles = self._get_compatible_packages(node) - node.conanfile.output.info(f"Main binary package '{original_package_id}' missing") - self._find_existing_compatible_binaries(node, compatibles, remotes, update) + if compatibles: + self._find_existing_compatible_binaries(node, compatibles, remotes, update) if node.binary == BINARY_MISSING and build_mode.allowed(node.conanfile): node.should_build = True node.build_allowed = True node.binary = BINARY_BUILD if not node.conanfile.info.cant_build else BINARY_INVALID + if node.binary == BINARY_INVALID and build_mode.allowed_compatible(node.conanfile): + if compatibles is None: + compatibles = self._get_compatible_packages(node) + if compatibles: + self._find_build_compatible_binary(node, compatibles) + if node.binary == BINARY_BUILD: conanfile = node.conanfile if conanfile.vendor and not conanfile.conf.get("tools.graph:vendor", choices=("build",)): diff --git a/test/integration/package_id/compatible_test.py b/test/integration/package_id/compatible_test.py index 5e27f7816f6..b7719e41711 100644 --- a/test/integration/package_id/compatible_test.py +++ b/test/integration/package_id/compatible_test.py @@ -448,3 +448,173 @@ def test_compatibility_msvc_and_cppstd(self): tc.run("create dep -pr=profile -s compiler.cppstd=20") tc.run("create . -pr=profile -s compiler.cppstd=17") tc.assert_listed_binary({"dep/1.0": ("b6d26a6bc439b25b434113982791edf9cab4d004", "Cache")}) + + +class TestCompatibleBuild: + def test_build_compatible(self): + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.build import check_min_cppstd + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + settings = "os", "compiler" + + def validate(self): + check_min_cppstd(self, 14) + """) + c.save({"conanfile.py": conanfile}) + settings = "-s os=Windows -s compiler=gcc -s compiler.version=11 " \ + "-s compiler.libcxx=libstdc++11 -s compiler.cppstd=11" + c.run(f"create . {settings}", assert_error=True) + c.assert_listed_binary({"pkg/0.1": ("bb33db23c961978d08dc0cdd6bc786b45b3e5943", "Invalid")}) + assert "pkg/0.1: Invalid: Current cppstd (11)" in c.out + + c.run(f"create . {settings} --build=compatible:&") + # the one for cppstd=14 is built!! + c.assert_listed_binary({"pkg/0.1": ("389803bed06200476fcee1af2023d4e9bfa24ff9", "Build")}) + c.run("list *:*") + assert "compiler.cppstd: 14" in c.out + + def test_build_compatible_cant_build(self): + # requires c++17 to build, can be consumed with c++14 + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.build import check_min_cppstd + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + settings = "os", "compiler" + + def validate(self): + check_min_cppstd(self, 14) + + def validate_build(self): + check_min_cppstd(self, 17) + """) + c.save({"conanfile.py": conanfile}) + settings = "-s os=Windows -s compiler=gcc -s compiler.version=11 " \ + "-s compiler.libcxx=libstdc++11 -s compiler.cppstd=11" + c.run(f"create . {settings}", assert_error=True) + c.assert_listed_binary({"pkg/0.1": ("bb33db23c961978d08dc0cdd6bc786b45b3e5943", "Invalid")}) + assert "pkg/0.1: Invalid: Current cppstd (11)" in c.out + + c.run(f"create . {settings} --build=missing", assert_error=True) + c.assert_listed_binary({"pkg/0.1": ("bb33db23c961978d08dc0cdd6bc786b45b3e5943", "Invalid")}) + assert "pkg/0.1: Invalid: Current cppstd (11)" in c.out + + c.run(f"create . {settings} --build=compatible:&") + # the one for cppstd=17 is built!! + c.assert_listed_binary({"pkg/0.1": ("58fb8ac6c2dc3e3f837253ce1a6ea59011525866", "Build")}) + c.run("list *:*") + assert "compiler.cppstd: 17" in c.out + + def test_build_compatible_cant_build2(self): + # requires c++17 to build, can be consumed with c++11 + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.build import check_min_cppstd + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + settings = "os", "compiler" + + def validate(self): + check_min_cppstd(self, 11) + + def validate_build(self): + check_min_cppstd(self, 17) + """) + c.save({"conanfile.py": conanfile}) + settings = "-s os=Windows -s compiler=gcc -s compiler.version=11 " \ + "-s compiler.libcxx=libstdc++11 -s compiler.cppstd=11" + c.run(f"create . {settings}", assert_error=True) + c.assert_listed_binary({"pkg/0.1": ("bb33db23c961978d08dc0cdd6bc786b45b3e5943", "Invalid")}) + assert "pkg/0.1: Cannot build for this configuration: Current cppstd (11)" in c.out + + c.run(f"create . {settings} --build=missing", assert_error=True) + # the one for cppstd=17 is built!! + c.assert_listed_binary({"pkg/0.1": ("bb33db23c961978d08dc0cdd6bc786b45b3e5943", "Invalid")}) + assert "pkg/0.1: Cannot build for this configuration: Current cppstd (11)" in c.out + + c.run(f"create . {settings} --build=compatible:&") + # the one for cppstd=17 is built!! + c.assert_listed_binary({"pkg/0.1": ("58fb8ac6c2dc3e3f837253ce1a6ea59011525866", "Build")}) + c.run("list *:*") + assert "compiler.cppstd: 17" in c.out + + def test_build_compatible_cant_build_only(self): + # requires c++17 to build, but don't specify consumption + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.build import check_min_cppstd + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + settings = "os", "compiler" + + def validate_build(self): + check_min_cppstd(self, 17) + """) + c.save({"conanfile.py": conanfile}) + settings = "-s os=Windows -s compiler=gcc -s compiler.version=11 " \ + "-s compiler.libcxx=libstdc++11 -s compiler.cppstd=11" + c.run(f"create . {settings}", assert_error=True) + c.assert_listed_binary({"pkg/0.1": ("bb33db23c961978d08dc0cdd6bc786b45b3e5943", "Invalid")}) + assert "pkg/0.1: Cannot build for this configuration: Current cppstd (11)" in c.out + + c.run(f"create . {settings} --build=missing", assert_error=True) + # the one for cppstd=17 is built!! + c.assert_listed_binary({"pkg/0.1": ("bb33db23c961978d08dc0cdd6bc786b45b3e5943", "Invalid")}) + assert "pkg/0.1: Cannot build for this configuration: Current cppstd (11)" in c.out + + c.run(f"create . {settings} --build=compatible:&") + # the one for cppstd=17 is built!! + c.assert_listed_binary({"pkg/0.1": ("58fb8ac6c2dc3e3f837253ce1a6ea59011525866", "Build")}) + c.run("list *:*") + assert "compiler.cppstd: 17" in c.out + + def test_multi_level_build_compatible(self): + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.build import check_min_cppstd + + class Pkg(ConanFile): + name = "{name}" + version = "0.1" + settings = "os", "compiler" + {requires} + + def validate(self): + check_min_cppstd(self, {cppstd}) + """) + c.save({"liba/conanfile.py": conanfile.format(name="liba", cppstd=14, requires=""), + "libb/conanfile.py": conanfile.format(name="libb", cppstd=17, + requires='requires="liba/0.1"')}) + c.run("export liba") + c.run("export libb") + settings = "-s os=Windows -s compiler=gcc -s compiler.version=11 " \ + "-s compiler.libcxx=libstdc++11 -s compiler.cppstd=11" + c.run(f"install --requires=libb/0.1 {settings}", assert_error=True) + c.assert_listed_binary({"liba/0.1": ("bb33db23c961978d08dc0cdd6bc786b45b3e5943", "Invalid"), + "libb/0.1": ("144910d65b27bcbf7d544201f5578555bbd0376e", "Invalid")}) + assert "liba/0.1: Invalid: Current cppstd (11)" in c.out + assert "libb/0.1: Invalid: Current cppstd (11)" in c.out + + c.run(f"install --requires=libb/0.1 {settings} --build=compatible") + # the one for cppstd=14 is built!! + c.assert_listed_binary({"liba/0.1": ("389803bed06200476fcee1af2023d4e9bfa24ff9", "Build"), + "libb/0.1": ("8f29f49be3ba2b6cbc9fa1e05432ce928b96ae5d", "Build")}) + c.run("list liba:*") + assert "compiler.cppstd: 14" in c.out + c.run("list libb:*") + assert "compiler.cppstd: 17" in c.out