From 613c6b3039fc6a66575bfb577110bbcd42458f30 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 2 Oct 2024 00:31:17 +0200 Subject: [PATCH 1/4] wip --- conan/cli/commands/graph.py | 8 +++++- conans/client/graph/install_graph.py | 28 +++++++++++++------ .../command_v2/test_info_build_order.py | 20 +++++++++++++ 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/conan/cli/commands/graph.py b/conan/cli/commands/graph.py index a929f39f1e9..32e5e71358c 100644 --- a/conan/cli/commands/graph.py +++ b/conan/cli/commands/graph.py @@ -97,6 +97,12 @@ def graph_build_order(conan_api, parser, subparser, *args): partial=args.lockfile_partial, overrides=overrides) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) + pr_args = [] + for context in "host", "build": + for f in "profile", "settings", "options", "conf": + s = "pr" if f == "profile" else f[0] + pr_args += [f'-{s}:{context[0]} "{v}"' for v in getattr(args, f"{f}_{context}") or []] + profile_args = " ".join(pr_args) if path: deps_graph = conan_api.graph.load_graph_consumer(path, args.name, args.version, @@ -116,7 +122,7 @@ def graph_build_order(conan_api, parser, subparser, *args): out = ConanOutput() out.title("Computing the build order") - install_graph = InstallGraph(deps_graph, order_by=args.order_by) + install_graph = InstallGraph(deps_graph, order_by=args.order_by, profile_args=profile_args) if args.reduce: if args.order_by is None: raise ConanException("--reduce needs --order-by argument defined") diff --git a/conans/client/graph/install_graph.py b/conans/client/graph/install_graph.py index 63221cc3238..9f9d96dd88d 100644 --- a/conans/client/graph/install_graph.py +++ b/conans/client/graph/install_graph.py @@ -335,13 +335,15 @@ class InstallGraph: """ A graph containing the package references in order to be built/downloaded """ - def __init__(self, deps_graph, order_by=None): + def __init__(self, deps_graph, order_by=None, profile_args=None): self._nodes = {} # ref with rev: _InstallGraphNode order_by = order_by or "recipe" self._order = order_by self._node_cls = _InstallRecipeReference if order_by == "recipe" else _InstallConfiguration self._is_test_package = False self.reduced = False + self._profile_args = profile_args + self._filename = None if deps_graph is not None: self._initialize_deps_graph(deps_graph) self._is_test_package = deps_graph.root.conanfile.tested_reference_str is not None @@ -368,15 +370,20 @@ def merge(self, other): self._nodes[ref] = install_node else: existing.merge(install_node) + if not isinstance(self._profile_args, dict): + self._profile_args = {self._filename: self._profile_args} + self._profile_args[other._filename] = other._profile_args @staticmethod def deserialize(data, filename): legacy = isinstance(data, list) - order, data, reduced = ("recipe", data, False) if legacy else \ - (data["order_by"], data["order"], data["reduced"]) + order, data, reduced, profile_args = ("recipe", data, False, None) if legacy else \ + (data["order_by"], data["order"], data["reduced"], data.get("profile_args")) result = InstallGraph(None, order_by=order) + result._filename = filename result.reduced = reduced result.legacy = legacy + result._profile_args = profile_args for level in data: for item in level: elem = result._node_cls.deserialize(item, filename) @@ -462,16 +469,18 @@ def install_build_order(self): result = {"order_by": self._order, "reduced": self.reduced, "order": [[n.serialize() for n in level] for level in install_order]} + if self._profile_args: + result["profile_args"] = self._profile_args return result def _get_missing_invalid_packages(self): missing, invalid = [], [] - def analyze_package(package): - if package.binary == BINARY_MISSING: - missing.append(package) - elif package.binary == BINARY_INVALID: - invalid.append(package) + def analyze_package(pkg): + if pkg.binary == BINARY_MISSING: + missing.append(pkg) + elif pkg.binary == BINARY_INVALID: + invalid.append(pkg) for _, install_node in self._nodes.items(): if self._order == "recipe": for package in install_node.packages.values(): @@ -505,7 +514,8 @@ def get_errors(self): return "\n".join(errors) return None - def _raise_invalid(self, invalid): + @staticmethod + def _raise_invalid(invalid): msg = ["There are invalid packages:"] for package in invalid: node = package.nodes[0] diff --git a/test/integration/command_v2/test_info_build_order.py b/test/integration/command_v2/test_info_build_order.py index 21f6b86a4d9..e8f3f6fe04c 100644 --- a/test/integration/command_v2/test_info_build_order.py +++ b/test/integration/command_v2/test_info_build_order.py @@ -765,3 +765,23 @@ def validate(self): tc.run("graph build-order-merge --file=order.json --file=order.json --format=json", assert_error=True) assert "dep/1.0:da39a3ee5e6b4b0d3255bfef95601890afd80709: Invalid configuration" in tc.out assert "IndexError: list index out of range" not in tc.out + + +def test_multi_configuration_profile_args(): + c = TestClient() + c.save({"pkg/conanfile.py": GenConanfile().with_settings("os"), + "consumer/conanfile.txt": "[requires]\npkg/0.1", + "mypr": ""}) + c.run("export pkg --name=pkg --version=0.1") + args = "-pr=mypr -s:b os=Linux -o:h *:shared=True -c:h user.my:conf=1" + c.run(f"graph build-order consumer --format=json --build=missing -s os=Windows {args} " + "--order-by=recipe", redirect_stdout="bo_win.json") + c.run(f"graph build-order consumer --format=json --build=missing -s os=Linux {args} " + "--order-by=recipe", redirect_stdout="bo_nix.json") + c.run("graph build-order-merge --file=bo_win.json --file=bo_nix.json --format=json", + redirect_stdout="bo3.json") + + bo_json = json.loads(c.load("bo3.json")) + win = '-pr:h "mypr" -s:h "os=Windows" -o:h "*:shared=True" -c:h "user.my:conf=1" -s:b "os=Linux"' + nix = '-pr:h "mypr" -s:h "os=Linux" -o:h "*:shared=True" -c:h "user.my:conf=1" -s:b "os=Linux"' + assert bo_json["profile_args"] == {"bo_win": win, "bo_nix": nix} From 3c031d3be612f715d434f46b961674e2bb369c18 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 2 Oct 2024 17:33:05 +0200 Subject: [PATCH 2/4] proposed profile args for build-order --- conan/cli/commands/graph.py | 11 ++--- conans/client/graph/install_graph.py | 48 ++++++++++++++----- .../command_v2/test_info_build_order.py | 3 +- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/conan/cli/commands/graph.py b/conan/cli/commands/graph.py index 32e5e71358c..430bad75ec9 100644 --- a/conan/cli/commands/graph.py +++ b/conan/cli/commands/graph.py @@ -14,7 +14,7 @@ from conan.errors import ConanException from conan.internal.deploy import do_deploys from conans.client.graph.graph import BINARY_MISSING -from conans.client.graph.install_graph import InstallGraph +from conans.client.graph.install_graph import InstallGraph, ProfileArgs from conans.errors import NotFoundException from conans.model.recipe_ref import ref_matches, RecipeReference @@ -97,12 +97,6 @@ def graph_build_order(conan_api, parser, subparser, *args): partial=args.lockfile_partial, overrides=overrides) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) - pr_args = [] - for context in "host", "build": - for f in "profile", "settings", "options", "conf": - s = "pr" if f == "profile" else f[0] - pr_args += [f'-{s}:{context[0]} "{v}"' for v in getattr(args, f"{f}_{context}") or []] - profile_args = " ".join(pr_args) if path: deps_graph = conan_api.graph.load_graph_consumer(path, args.name, args.version, @@ -122,7 +116,8 @@ def graph_build_order(conan_api, parser, subparser, *args): out = ConanOutput() out.title("Computing the build order") - install_graph = InstallGraph(deps_graph, order_by=args.order_by, profile_args=profile_args) + install_graph = InstallGraph(deps_graph, order_by=args.order_by, + profile_args=ProfileArgs.from_args(args)) if args.reduce: if args.order_by is None: raise ConanException("--reduce needs --order-by argument defined") diff --git a/conans/client/graph/install_graph.py b/conans/client/graph/install_graph.py index 9f9d96dd88d..ab99c92569b 100644 --- a/conans/client/graph/install_graph.py +++ b/conans/client/graph/install_graph.py @@ -331,6 +331,28 @@ def merge(self, other): self.filenames.append(d) +class ProfileArgs: + def __init__(self, args): + self._args = args + + @staticmethod + def from_args(args): + pr_args = [] + for context in "host", "build": + for f in "profile", "settings", "options", "conf": + s = "pr" if f == "profile" else f[0] + pr_args += [f'-{s}:{context[0]} "{v}"' for v in + getattr(args, f"{f}_{context}") or []] + return ProfileArgs(" ".join(pr_args)) + + @staticmethod + def deserialize(data): + return ProfileArgs(data.get("args")) + + def serialize(self): + return {"args": self._args} + + class InstallGraph: """ A graph containing the package references in order to be built/downloaded """ @@ -342,7 +364,7 @@ def __init__(self, deps_graph, order_by=None, profile_args=None): self._node_cls = _InstallRecipeReference if order_by == "recipe" else _InstallConfiguration self._is_test_package = False self.reduced = False - self._profile_args = profile_args + self._profiles = {"self": profile_args} if profile_args is not None else {} self._filename = None if deps_graph is not None: self._initialize_deps_graph(deps_graph) @@ -370,20 +392,24 @@ def merge(self, other): self._nodes[ref] = install_node else: existing.merge(install_node) - if not isinstance(self._profile_args, dict): - self._profile_args = {self._filename: self._profile_args} - self._profile_args[other._filename] = other._profile_args + # Make sure that self is also updated + current = self._profiles.pop("self", None) + if current is not None: + self._profiles[self._filename] = current + new = other._profiles.get("self") + if new is not None: + self._profiles[other._filename] = new @staticmethod def deserialize(data, filename): legacy = isinstance(data, list) - order, data, reduced, profile_args = ("recipe", data, False, None) if legacy else \ - (data["order_by"], data["order"], data["reduced"], data.get("profile_args")) + order, data, reduced, profiles = ("recipe", data, False, {}) if legacy else \ + (data["order_by"], data["order"], data["reduced"], data.get("profiles", {})) result = InstallGraph(None, order_by=order) - result._filename = filename result.reduced = reduced result.legacy = legacy - result._profile_args = profile_args + result._filename = filename + result._profiles = {k: ProfileArgs.deserialize(v) for k, v in profiles.items()} for level in data: for item in level: elem = result._node_cls.deserialize(item, filename) @@ -468,9 +494,9 @@ def install_build_order(self): install_order = self.install_order() result = {"order_by": self._order, "reduced": self.reduced, - "order": [[n.serialize() for n in level] for level in install_order]} - if self._profile_args: - result["profile_args"] = self._profile_args + "order": [[n.serialize() for n in level] for level in install_order], + "profiles": {k: v.serialize() for k, v in self._profiles.items()} + } return result def _get_missing_invalid_packages(self): diff --git a/test/integration/command_v2/test_info_build_order.py b/test/integration/command_v2/test_info_build_order.py index e8f3f6fe04c..16cd8f1b1db 100644 --- a/test/integration/command_v2/test_info_build_order.py +++ b/test/integration/command_v2/test_info_build_order.py @@ -780,8 +780,7 @@ def test_multi_configuration_profile_args(): "--order-by=recipe", redirect_stdout="bo_nix.json") c.run("graph build-order-merge --file=bo_win.json --file=bo_nix.json --format=json", redirect_stdout="bo3.json") - bo_json = json.loads(c.load("bo3.json")) win = '-pr:h "mypr" -s:h "os=Windows" -o:h "*:shared=True" -c:h "user.my:conf=1" -s:b "os=Linux"' nix = '-pr:h "mypr" -s:h "os=Linux" -o:h "*:shared=True" -c:h "user.my:conf=1" -s:b "os=Linux"' - assert bo_json["profile_args"] == {"bo_win": win, "bo_nix": nix} + assert bo_json["profiles"] == {"bo_win": {"args": win}, "bo_nix": {"args": nix}} From a988c67674652eb65e99a6d0551567be9e0fb81f Mon Sep 17 00:00:00 2001 From: James Date: Tue, 8 Oct 2024 11:46:05 +0200 Subject: [PATCH 3/4] Update conans/client/graph/install_graph.py --- conans/client/graph/install_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/client/graph/install_graph.py b/conans/client/graph/install_graph.py index ab99c92569b..bacfb81284f 100644 --- a/conans/client/graph/install_graph.py +++ b/conans/client/graph/install_graph.py @@ -341,7 +341,7 @@ def from_args(args): for context in "host", "build": for f in "profile", "settings", "options", "conf": s = "pr" if f == "profile" else f[0] - pr_args += [f'-{s}:{context[0]} "{v}"' for v in + pr_args += [f'-{s}:{context[0]}="{v}"' for v in getattr(args, f"{f}_{context}") or []] return ProfileArgs(" ".join(pr_args)) From 6c1a3d52a9d05860dc1b28bbb0a0976dc0b2da90 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 8 Oct 2024 14:12:54 +0200 Subject: [PATCH 4/4] review --- test/integration/command_v2/test_info_build_order.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/command_v2/test_info_build_order.py b/test/integration/command_v2/test_info_build_order.py index 16cd8f1b1db..7ebfcd47739 100644 --- a/test/integration/command_v2/test_info_build_order.py +++ b/test/integration/command_v2/test_info_build_order.py @@ -781,6 +781,6 @@ def test_multi_configuration_profile_args(): c.run("graph build-order-merge --file=bo_win.json --file=bo_nix.json --format=json", redirect_stdout="bo3.json") bo_json = json.loads(c.load("bo3.json")) - win = '-pr:h "mypr" -s:h "os=Windows" -o:h "*:shared=True" -c:h "user.my:conf=1" -s:b "os=Linux"' - nix = '-pr:h "mypr" -s:h "os=Linux" -o:h "*:shared=True" -c:h "user.my:conf=1" -s:b "os=Linux"' + win = '-pr:h="mypr" -s:h="os=Windows" -o:h="*:shared=True" -c:h="user.my:conf=1" -s:b="os=Linux"' + nix = '-pr:h="mypr" -s:h="os=Linux" -o:h="*:shared=True" -c:h="user.my:conf=1" -s:b="os=Linux"' assert bo_json["profiles"] == {"bo_win": {"args": win}, "bo_nix": {"args": nix}}