From 70f9c168601d7c946f546c627d923d0612cad119 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 17 May 2024 14:09:53 +0200 Subject: [PATCH 1/8] Report fully rendered recipe to stdout --- conda_build/api.py | 58 +++++++------------------------------------ conda_build/build.py | 9 +++++++ conda_build/render.py | 54 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 50 deletions(-) diff --git a/conda_build/api.py b/conda_build/api.py index cc866a865d..162805edf8 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -52,11 +52,7 @@ def render( templates evaluated. Returns a list of (metadata, need_download, need_reparse in env) tuples""" - - from conda.exceptions import NoPackagesFoundError - - from .exceptions import DependencyNeedsBuildingError - from .render import finalize_metadata, render_recipe + from .render import render_metadata_tuples, render_recipe config = get_or_merge_config(config, **kwargs) @@ -68,50 +64,14 @@ def render( variants=variants, permit_unsatisfiable_variants=permit_unsatisfiable_variants, ) - output_metas: dict[tuple[str, str, tuple[tuple[str, str], ...]], MetaDataTuple] = {} - for meta, download, render_in_env in metadata_tuples: - if not meta.skip() or not config.trim_skip: - for od, om in meta.get_output_metadata_set( - permit_unsatisfiable_variants=permit_unsatisfiable_variants, - permit_undefined_jinja=not finalize, - bypass_env_check=bypass_env_check, - ): - if not om.skip() or not config.trim_skip: - if "type" not in od or od["type"] == "conda": - if finalize and not om.final: - try: - om = finalize_metadata( - om, - permit_unsatisfiable_variants=permit_unsatisfiable_variants, - ) - except (DependencyNeedsBuildingError, NoPackagesFoundError): - if not permit_unsatisfiable_variants: - raise - - # remove outputs section from output objects for simplicity - if not om.path and (outputs := om.get_section("outputs")): - om.parent_outputs = outputs - del om.meta["outputs"] - - output_metas[ - om.dist(), - om.config.variant.get("target_platform"), - tuple( - (var, om.config.variant[var]) - for var in om.get_used_vars() - ), - ] = MetaDataTuple(om, download, render_in_env) - else: - output_metas[ - f"{om.type}: {om.name()}", - om.config.variant.get("target_platform"), - tuple( - (var, om.config.variant[var]) - for var in om.get_used_vars() - ), - ] = MetaDataTuple(om, download, render_in_env) - - return list(output_metas.values()) + return render_metadata_tuples( + metadata_tuples, + config=config, + permit_unsatisfiable_variants=permit_unsatisfiable_variants, + finalize=finalize, + bypass_env_check=bypass_env_check, + ) + def output_yaml( diff --git a/conda_build/build.py b/conda_build/build.py index 6dd2b49256..143998d0df 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -59,6 +59,7 @@ execute_download_actions, expand_outputs, output_yaml, + render_metadata_tuples, render_recipe, reparse, try_download, @@ -2425,6 +2426,14 @@ def build( # Write out metadata for `conda debug`, making it obvious that this is what it is, must be done # after try_download() output_yaml(m, os.path.join(m.config.work_dir, "metadata_conda_debug.yaml")) + if m.config.verbose: + m_copy = m.copy() + for om, _, _ in render_metadata_tuples([(m_copy, False, False)], m_copy.config): + print("", "Rendered as:", "```yaml", output_yaml(om).rstrip(), "```", "", sep="\n") + # Each iteration returns the whole meta yaml, and then we are supposed to remove + # the outputs we don't want. Instead we just take the first and print it fully + break + del m_copy # get_dir here might be just work, or it might be one level deeper, # dependening on the source. diff --git a/conda_build/render.py b/conda_build/render.py index cc3bcd87c0..72260b1a14 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -27,7 +27,7 @@ from conda.base.context import context from conda.cli.common import specs_from_url from conda.core.package_cache_data import ProgressiveFetchExtract -from conda.exceptions import UnsatisfiableError +from conda.exceptions import NoPackagesFoundError, UnsatisfiableError from conda.gateways.disk.create import TemporaryDirectory from conda.models.records import PackageRecord from conda.models.version import VersionOrder @@ -999,6 +999,58 @@ def render_recipe( bypass_env_check=bypass_env_check, ) +def render_metadata_tuples( + metadata_tuples: Iterable[MetaDataTuple], + config: Config, + permit_unsatisfiable_variants: bool = True, + finalize: bool = True, + bypass_env_check: bool = False +) -> list[MetaDataTuple]: + output_metas: dict[tuple[str, str, tuple[tuple[str, str], ...]], MetaDataTuple] = {} + for meta, download, render_in_env in metadata_tuples: + if not meta.skip() or not config.trim_skip: + for od, om in meta.get_output_metadata_set( + permit_unsatisfiable_variants=permit_unsatisfiable_variants, + permit_undefined_jinja=not finalize, + bypass_env_check=bypass_env_check, + ): + if not om.skip() or not config.trim_skip: + if "type" not in od or od["type"] == "conda": + if finalize and not om.final: + try: + om = finalize_metadata( + om, + permit_unsatisfiable_variants=permit_unsatisfiable_variants, + ) + except (DependencyNeedsBuildingError, NoPackagesFoundError): + if not permit_unsatisfiable_variants: + raise + + # remove outputs section from output objects for simplicity + if not om.path and (outputs := om.get_section("outputs")): + om.parent_outputs = outputs + del om.meta["outputs"] + + output_metas[ + om.dist(), + om.config.variant.get("target_platform"), + tuple( + (var, om.config.variant[var]) + for var in om.get_used_vars() + ), + ] = MetaDataTuple(om, download, render_in_env) + else: + output_metas[ + f"{om.type}: {om.name()}", + om.config.variant.get("target_platform"), + tuple( + (var, om.config.variant[var]) + for var in om.get_used_vars() + ), + ] = MetaDataTuple(om, download, render_in_env) + + return list(output_metas.values()) + # Keep this out of the function below so it can be imported by other modules. FIELDS = [ From 206c006807a5d4e5ec1b2afefc337265bf506b4c Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 17 May 2024 14:37:14 +0200 Subject: [PATCH 2/8] add test --- tests/test_api_build.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_api_build.py b/tests/test_api_build.py index a663f18e73..c8188ef9f8 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -1968,3 +1968,14 @@ def test_add_pip_as_python_dependency_from_condarc_file( with env_var("CONDARC", conda_rc, reset_context): with check_build_fails: api.build(testing_metadata) + + +def test_rendered_is_reported(testing_config, capsys): + recipe_dir = os.path.join(metadata_dir, "outputs_overwrite_base_file") + api.build(recipe_dir, config=testing_config) + + captured = capsys.readouterr() + assert "Rendered as:" in captured.out + assert "name: base-outputs_overwrite_base_file" in captured.out + assert "- name: base-outputs_overwrite_base_file" in captured.out + assert "- base-outputs_overwrite_base_file >=1.0,<2.0a0" in captured.out From c245ef2f3b65c44f79d4a6707d9b1cf3695f949d Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 17 May 2024 14:39:16 +0200 Subject: [PATCH 3/8] add news --- news/5344-report-rendered | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 news/5344-report-rendered diff --git a/news/5344-report-rendered b/news/5344-report-rendered new file mode 100644 index 0000000000..7dccf4b960 --- /dev/null +++ b/news/5344-report-rendered @@ -0,0 +1,19 @@ +### Enhancements + +* Report fully rendered recipe to stdout before the build process starts. (#3798 via #5344) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* From 81546908f50b3d394c6aa3d64f469804a79249e8 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 17 May 2024 14:54:36 +0200 Subject: [PATCH 4/8] pre-commit --- conda_build/api.py | 1 - conda_build/build.py | 14 ++++++++++++-- conda_build/render.py | 3 ++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/conda_build/api.py b/conda_build/api.py index 162805edf8..eaea8f50b8 100644 --- a/conda_build/api.py +++ b/conda_build/api.py @@ -73,7 +73,6 @@ def render( ) - def output_yaml( metadata: MetaData, file_path: str | os.PathLike | Path | None = None, diff --git a/conda_build/build.py b/conda_build/build.py index 143998d0df..eb8bdfecd1 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -2428,8 +2428,18 @@ def build( output_yaml(m, os.path.join(m.config.work_dir, "metadata_conda_debug.yaml")) if m.config.verbose: m_copy = m.copy() - for om, _, _ in render_metadata_tuples([(m_copy, False, False)], m_copy.config): - print("", "Rendered as:", "```yaml", output_yaml(om).rstrip(), "```", "", sep="\n") + for om, _, _ in render_metadata_tuples( + [(m_copy, False, False)], m_copy.config + ): + print( + "", + "Rendered as:", + "```yaml", + output_yaml(om).rstrip(), + "```", + "", + sep="\n", + ) # Each iteration returns the whole meta yaml, and then we are supposed to remove # the outputs we don't want. Instead we just take the first and print it fully break diff --git a/conda_build/render.py b/conda_build/render.py index 72260b1a14..496e7fec2b 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -999,12 +999,13 @@ def render_recipe( bypass_env_check=bypass_env_check, ) + def render_metadata_tuples( metadata_tuples: Iterable[MetaDataTuple], config: Config, permit_unsatisfiable_variants: bool = True, finalize: bool = True, - bypass_env_check: bool = False + bypass_env_check: bool = False, ) -> list[MetaDataTuple]: output_metas: dict[tuple[str, str, tuple[tuple[str, str], ...]], MetaDataTuple] = {} for meta, download, render_in_env in metadata_tuples: From 83260cebf7ddb1235c46f24cf62637abc21fa472 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 27 May 2024 19:10:28 +0000 Subject: [PATCH 5/8] Do not use f-strings here, it might be executed by older python versions --- conda_build/_load_setup_py_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/_load_setup_py_data.py b/conda_build/_load_setup_py_data.py index 9180c404fc..bd520fde0f 100644 --- a/conda_build/_load_setup_py_data.py +++ b/conda_build/_load_setup_py_data.py @@ -111,7 +111,7 @@ def setup(**kw): exec(code, ns, ns) else: if not permit_undefined_jinja: - raise TypeError(f"{setup_file} is not a file that can be read") + raise TypeError("%s is not a file that can be read" % setup_file) sys.modules["versioneer"] = versioneer From 2110719dd36d82fdd2a3b5b2e2cb0f20027ed775 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 27 May 2024 19:18:15 +0000 Subject: [PATCH 6/8] ignore ruff --- conda_build/_load_setup_py_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/_load_setup_py_data.py b/conda_build/_load_setup_py_data.py index bd520fde0f..b2d8d0731b 100644 --- a/conda_build/_load_setup_py_data.py +++ b/conda_build/_load_setup_py_data.py @@ -111,7 +111,7 @@ def setup(**kw): exec(code, ns, ns) else: if not permit_undefined_jinja: - raise TypeError("%s is not a file that can be read" % setup_file) + raise TypeError("%s is not a file that can be read" % setup_file) # noqa: UP031 sys.modules["versioneer"] = versioneer From ddb7436822c9788d676251b4338835b837f0ddb4 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 27 May 2024 19:40:28 +0000 Subject: [PATCH 7/8] catch exceptions here too --- conda_build/jinja_context.py | 6 +++++- tests/test_api_build.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 6ec2195eb0..16b0c5c9ff 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -10,6 +10,7 @@ import time from functools import partial from io import StringIO, TextIOBase +from subprocess import CalledProcessError from typing import TYPE_CHECKING from warnings import warn @@ -165,7 +166,10 @@ def load_setup_py_data( args.extend(["--recipe-dir", recipe_dir]) if permit_undefined_jinja: args.append("--permit-undefined-jinja") - check_call_env(args, env=env) + try: + check_call_env(args, env=env) + except CalledProcessError as exc: + raise CondaBuildException("Could not run load_setup_py_data in subprocess.") from exc # this is a file that the subprocess will have written with open( os.path.join(m.config.work_dir, "conda_build_loaded_setup_py.json") diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 2e3f0aaab8..6087c8d3c1 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -1540,7 +1540,7 @@ def test_setup_py_data_in_env(testing_config): # should pass with any modern python (just not 3.5) api.build(recipe, config=testing_config) # make sure it fails with our special python logic - with pytest.raises(BuildScriptException): + with pytest.raises((BuildScriptException, CondaBuildException)): api.build(recipe, config=testing_config, python="3.5") From 12fa5189ac0a94c166738e36bf00da7af7ea3eff Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 27 May 2024 19:46:29 +0000 Subject: [PATCH 8/8] format --- conda_build/jinja_context.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 16b0c5c9ff..307a13ecc9 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -169,7 +169,9 @@ def load_setup_py_data( try: check_call_env(args, env=env) except CalledProcessError as exc: - raise CondaBuildException("Could not run load_setup_py_data in subprocess.") from exc + raise CondaBuildException( + "Could not run load_setup_py_data in subprocess" + ) from exc # this is a file that the subprocess will have written with open( os.path.join(m.config.work_dir, "conda_build_loaded_setup_py.json")