Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add finalize() method to ConanFile #16646

Merged
merged 53 commits into from
Aug 27, 2024
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
53 commits
Select commit Hold shift + click to select a range
feec493
Sketch first approach for install method in ConanFile
AbrilRBS Jul 10, 2024
0affd24
Refactor
AbrilRBS Jul 10, 2024
b3f20c6
Refactor
AbrilRBS Jul 10, 2024
bb9d775
Typos
AbrilRBS Jul 10, 2024
cdbebfb
Refactor tests
AbrilRBS Jul 10, 2024
f84663d
Merge branch 'develop2' into ar/install-method
AbrilRBS Jul 10, 2024
1d4d44c
Improve tests, start to think about editables
AbrilRBS Jul 10, 2024
2d41063
Minimal tool_requires test
AbrilRBS Jul 10, 2024
4fee15f
Add failing tool_require test, cpp_info path releated
AbrilRBS Jul 10, 2024
c33186e
Make package_info understand installed package
AbrilRBS Jul 11, 2024
9f086d2
Use package_folder instead of cheking again install method
perseoGI Jul 11, 2024
f2843bf
Fixes and additions, do not support editables
AbrilRBS Jul 12, 2024
5ac885f
Update conans/model/layout.py
AbrilRBS Jul 22, 2024
0b36d5e
Update conans/model/layout.py
AbrilRBS Jul 22, 2024
0194c40
Update conans/model/layout.py
AbrilRBS Jul 22, 2024
b3f0ec4
Update conans/client/installer.py
AbrilRBS Jul 22, 2024
e88936a
Update conans/model/layout.py
AbrilRBS Jul 22, 2024
5f19137
Update conans/model/layout.py
AbrilRBS Jul 22, 2024
3a4890a
Merge branch 'develop2' into ar/install-method
AbrilRBS Jul 22, 2024
43f537d
Update test/integration/conanfile/test_install_method.py
AbrilRBS Jul 22, 2024
05eebfa
More tests
AbrilRBS Jul 22, 2024
821fec6
More tests
AbrilRBS Jul 22, 2024
830b922
Reorder, add new test
AbrilRBS Jul 22, 2024
cbb9ea5
First tool_requires test
AbrilRBS Jul 22, 2024
8a2c8b0
Fix test
AbrilRBS Jul 22, 2024
cf807af
Improve with_test api
AbrilRBS Jul 22, 2024
c1cfaed
Remove duplicate future test
AbrilRBS Jul 22, 2024
784e207
Add support for install() in conan cache path, add a few tests to sho…
AbrilRBS Jul 22, 2024
0019e7b
Better name for original_package_folder, fallback to package_folder w…
AbrilRBS Jul 22, 2024
42610cc
Codify install() not running in a restore
AbrilRBS Jul 22, 2024
73f5167
Add access to immutable_package_folder from conanfile/conanfileinterfaze
AbrilRBS Jul 22, 2024
1c89195
Add test comment
AbrilRBS Jul 22, 2024
0eeeff3
conan cache check-integrity test
AbrilRBS Jul 22, 2024
a2c93e4
Reorder
AbrilRBS Jul 22, 2024
6037bb3
Extra step for cache save/restore
AbrilRBS Jul 22, 2024
63fac5a
Add immutable access test from consumer
AbrilRBS Jul 22, 2024
308f2dc
More tests, one failing for package_id reasons, add logging for install
AbrilRBS Jul 23, 2024
993510f
Allow only info access in install()
AbrilRBS Jul 23, 2024
551b481
Typo
AbrilRBS Jul 23, 2024
6599fde
Fix Windows test
AbrilRBS Jul 23, 2024
3ff678c
More logging
AbrilRBS Jul 23, 2024
70d17cf
Add conan upload ... --check test
AbrilRBS Jul 23, 2024
0211903
Add new tests and serialize immutable_package folder in ConanFile
AbrilRBS Aug 2, 2024
c0a5d58
Extra check that it in fact would break
AbrilRBS Aug 2, 2024
65c2bdf
Simplify
AbrilRBS Aug 2, 2024
7abf734
Merge branch 'develop2' into ar/install-method
AbrilRBS Aug 2, 2024
c448709
Rename install() to localize(), remove conanfile.localize_folder
AbrilRBS Aug 12, 2024
c615e52
Rename method to finalize()
AbrilRBS Aug 20, 2024
fb2f4d1
Cleanup code
AbrilRBS Aug 20, 2024
b1b4d59
Discard changes to test/integration/conanfile/conanfile_errors_test.py
AbrilRBS Aug 20, 2024
2fdffb8
Merge branch 'develop2' into ar/install-method
AbrilRBS Aug 20, 2024
a134f60
Update conans/model/layout.py
memsharded Aug 20, 2024
0fd1196
Update conans/model/layout.py
memsharded Aug 20, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions conan/internal/cache/conan_reference_layout.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
SRC_FOLDER = "s"
BUILD_FOLDER = "b"
PACKAGES_FOLDER = "p"
INSTALL_FOLDER = "i"
EXPORT_FOLDER = "e"
EXPORT_SRC_FOLDER = "es"
DOWNLOAD_EXPORT_FOLDER = "d"
Expand Down Expand Up @@ -55,7 +56,7 @@ def export_sources(self):
return os.path.join(self._base_folder, EXPORT_SRC_FOLDER)

def metadata(self):
return os.path.join(self.download_export(), "metadata")
return os.path.join(self.download_export(), METADATA)

def download_export(self):
return os.path.join(self._base_folder, DOWNLOAD_EXPORT_FOLDER)
Expand Down Expand Up @@ -99,11 +100,14 @@ def build(self):
def package(self):
return os.path.join(self._base_folder, PACKAGES_FOLDER)

def install(self):
return os.path.join(self._base_folder, INSTALL_FOLDER)

def download_package(self):
return os.path.join(self._base_folder, DOWNLOAD_EXPORT_FOLDER)

def metadata(self):
return os.path.join(self.download_package(), "metadata")
return os.path.join(self.download_package(), METADATA)

def package_manifests(self):
package_folder = self.package()
Expand Down
27 changes: 26 additions & 1 deletion conan/test/assets/genconanfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ def __init__(self, name=None, version=None):
self._provides = None
self._deprecated = None
self._package_lines = None
self._install_lines = None
AbrilRBS marked this conversation as resolved.
Show resolved Hide resolved
self._package_files = None
self._package_files_env = None
self._package_files_link = None
Expand Down Expand Up @@ -207,6 +208,12 @@ def with_package(self, *lines):
self._package_lines.append(line)
return self

def with_install(self, *lines):
self._install_lines = self._install_lines or []
for line in lines:
self._install_lines.append(line)
return self

def with_build_msg(self, msg):
self._build_messages = self._build_messages or []
self._build_messages.append(msg)
Expand Down Expand Up @@ -360,6 +367,10 @@ def _package_method(self):
return (self._package_lines or self._package_files or self._package_files_env or
self._package_files_link)

@property
def _install_method(self):
return self._install_lines

@property
def _package_method_render(self):
lines = []
Expand Down Expand Up @@ -388,6 +399,19 @@ def package(self):
{}
""".format("\n".join(lines))

@property
def _install_method_render(self):
lines = []
if self._install_lines:
lines.extend(" {}".format(line) for line in self._install_lines)

if not lines:
return ""
return """
def install(self):
{}
""".format("\n".join(lines))

@property
def _build_render(self):
if not self._build_messages and not self._cmake_build:
Expand Down Expand Up @@ -463,7 +487,8 @@ def __repr__(self):
"exports_sources", "exports", "generators", "requires", "build_requires",
"tool_requires", "test_requires", "requirements", "python_requires",
"revision_mode", "settings", "options", "default_options", "build",
"package_method", "package_info", "package_id_lines", "test_lines"
"package_method", "package_info", "package_id_lines", "test_lines",
"install_method"
):
if member == "requirements":
# FIXME: This seems exclusive, but we could mix them?
Expand Down
14 changes: 13 additions & 1 deletion conans/client/installer.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,9 @@ def _handle_package(self, package, install_reference, handled_count, total_count
# Call the info method
conanfile.folders.set_base_package(pkg_folder)
conanfile.folders.set_base_pkg_metadata(pkg_metadata)
self._call_package_info(conanfile, pkg_folder, is_editable=False)
self._call_install_method(conanfile, package_layout.install())
AbrilRBS marked this conversation as resolved.
Show resolved Hide resolved
# Use package_folder which has been updated previously by install_method if necessary
self._call_package_info(conanfile, conanfile.package_folder, is_editable=False)

def _handle_node_editable(self, install_node):
# It will only run generation
Expand Down Expand Up @@ -454,3 +456,13 @@ def _call_package_info(self, conanfile, package_folder, is_editable):
self._hook_manager.execute("post_package_info", conanfile=conanfile)

conanfile.cpp_info.check_component_requires(conanfile)

def _call_install_method(self, conanfile, install_folder):
if hasattr(conanfile, "install"):
conanfile.folders.set_install_folder(install_folder)
AbrilRBS marked this conversation as resolved.
Show resolved Hide resolved
if not os.path.exists(install_folder):
AbrilRBS marked this conversation as resolved.
Show resolved Hide resolved
mkdir(install_folder)
with conanfile_exception_formatter(conanfile, "install"):
conanfile.install()
# TODO: Keep the orignal info in something like "invariante_package_folder"
AbrilRBS marked this conversation as resolved.
Show resolved Hide resolved
conanfile.folders.set_base_package(install_folder)
AbrilRBS marked this conversation as resolved.
Show resolved Hide resolved
AbrilRBS marked this conversation as resolved.
Show resolved Hide resolved
4 changes: 4 additions & 0 deletions conans/model/conan_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,10 @@ def package_folder(self):
"""
return self.folders.base_package

@property
def install_folder(self):
return self.folders.install_folder

@property
def generators_folder(self):
return self.folders.generators_folder
Expand Down
16 changes: 16 additions & 0 deletions conans/model/layout.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,13 @@ def __init__(self):

self._base_recipe_metadata = None
self._base_pkg_metadata = None
# TODO: Think about adding invariant package fodlder ref in case of install()
AbrilRBS marked this conversation as resolved.
Show resolved Hide resolved
self._original_package_folder = None

self.source = ""
self.build = ""
self.package = ""
self.install = ""
self.generators = ""
# Relative location of the project root, if the conanfile is not in that project root, but
# in a subfolder: e.g: If the conanfile is in a subfolder then self.root = ".."
Expand Down Expand Up @@ -142,6 +145,19 @@ def package_folder(self):
"""For the cache, the package folder is only the base"""
return self._base_package

def set_install_folder(self, folder):
# TODO: Store when the install folder is set, to be able to access the old package later
AbrilRBS marked this conversation as resolved.
Show resolved Hide resolved
self._original_package_folder = self.package_folder
self.install = folder

@property
def install_folder(self):
return self.install

@property
def original_package_folder(self):
return self._original_package_folder

@property
def generators_folder(self):
if self._base_generators is None:
Expand Down
184 changes: 184 additions & 0 deletions test/integration/conanfile/test_install_method.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
import os
import textwrap

import pytest

from conan.test.assets.genconanfile import GenConanfile

from conan.test.utils.tools import TestClient
from conans.util.files import load

conanfile_dep = textwrap.dedent("""
import os
from conan import ConanFile
from conan.tools.files import save, copy

class TestConan(ConanFile):
name = "dep"
version = "1.0"
def package(self):
save(self, os.path.join(self.package_folder, "file.txt"), "Hello World!")

def install(self):
self.output.info(f"Running install method in {self.install_folder}")
# copy(self, "*", src=self.package_folder, dst=self.install_folder)
copy(self, "file.txt", src=self.package_folder, dst=self.install_folder)
save(self, os.path.join(self.install_folder, "installed.txt"), "Installed file")

def package_info(self):
self.output.info(f"Running package_info method in {self.package_folder}")
""")


class TestBasicLocalFlows:

@pytest.fixture
def client(self):
tc = TestClient(light=True)
tc.save({"dep/conanfile.py": conanfile_dep})
tc.run("export dep")
return tc

def test_basic_install_method(self, client):
client.run("create dep")
layout = client.created_layout()
assert layout.package().endswith("p")
assert f"Package folder {layout.package()}" in client.out
assert f"Running install method in {layout.install()}" in client.out
assert f"Running package_info method in {layout.install()}" in client.out
client.run("install --requires=dep/1.0")
assert f"Running package_info method in {layout.install()}" in client.out
# Only issue is that the PackageLayout has no idea about the redirected package folder
# So we have to know to check for it in tests, but oh well
assert "installed.txt" in os.listdir(layout.install())
assert "installed.txt" not in os.listdir(layout.package())

def test_dependency_install_method(self, client):
client.save({"app/conanfile.py": textwrap.dedent("""
from conan import ConanFile
class TestConan(ConanFile):
name = "app"
version = "1.0"
requires = "dep/1.0"
def generate(self):
self.output.info("Running generate method")
dep_pkg_folder = self.dependencies["dep"].package_folder
self.output.info(f"Dep package folder: {dep_pkg_folder}")
""")})
client.run("create dep")
dep_layout = client.created_layout()
client.run("create app")
assert f"Dep package folder: {dep_layout.package()}" not in client.out
assert f"Dep package folder: {dep_layout.install()}" in client.out

def test_save_restore_cache(self):
pass

def test_lockfile_interaction(self):
pass

def test_remove_deletes_correct_folders(self):
pass

def test_graph_info_output(self):
# The output should serialize the orignal package folder path to give users the info
pass

def test_create_pkglist_output(self):
pass

def test_vendorized(self):
# TODO: Should this be handled? Or are vendoring packages meant to know if they are dealing with localized versions?
pass

AbrilRBS marked this conversation as resolved.
Show resolved Hide resolved

class TestToolRequiresFlows:
def test_tool_requires(self):
tc = TestClient(light=True)
tc.save({"dep/conanfile.py": textwrap.dedent("""
import os
from conan import ConanFile
from conan.tools.files import save, copy

class TestConan(ConanFile):
name = "dep"
version = "1.0"
package_type = "application"
def package(self):
save(self, os.path.join(self.package_folder, "bin", "executable.txt"), "Base")

def install(self):
self.output.info(f"Running install method in {self.install_folder}")
copy(self, "*", src=self.package_folder, dst=self.install_folder)
save(self, os.path.join(self.install_folder, "bin", "installed.txt"), "Installed file")

def package_info(self):
self.output.info(f"Running package_info method in {self.package_folder}")
self.cpp_info.bindirs = ["bin"]

"""), "app/conanfile.py": textwrap.dedent("""
from conan import ConanFile
import os

class TestConan(ConanFile):
name = "app"
version = "1.0"

def build_requirements(self):
self.tool_requires("dep/1.0")

def build(self):
self.output.info("Running build method")
bindir = self.dependencies.build['dep'].cpp_info.bindir
self.output.info(f"Dep bindir: {bindir}")
self.output.info(f"Is installed? {os.path.exists(os.path.join(bindir, 'installed.txt'))}")
""")})
tc.run("create dep --build-require")
dep_layout = tc.created_layout()
tc.run("create app")
# This fails. cpp_info is using the original package folder to construct the final path
assert f"Dep bindir: {dep_layout.install()}" in tc.out
assert "app/1.0: Is installed? True" in tc.out

def test_update_recipe(self):
pass


class TestRemoteFlows:

@pytest.fixture
def client(self):
tc = TestClient(light=True, default_server_user=True)
tc.save({"dep/conanfile.py": conanfile_dep})
tc.run("export dep")
return tc

def test_remote_upload_install_method(self, client):
client.run("create dep")
created_pref = client.created_package_reference("dep/1.0")
client.run("upload * -r=default -c")

# Only the package folder is uploaded, not the install folder
uploaded_pref_path = client.servers["default"].test_server.server_store.package(created_pref)
manifest_contents = load(os.path.join(uploaded_pref_path, "conanmanifest.txt"))
assert "file.txt" in manifest_contents
assert "installed.txt" not in manifest_contents

client.run("remove * -c")
client.run(f"download {created_pref} -r=default")
downloaded_pref_layout = client.get_latest_pkg_layout(created_pref)
assert "file.txt" in os.listdir(downloaded_pref_layout.package())
assert "installed.txt" not in os.listdir(downloaded_pref_layout.package())
assert not os.path.exists(os.path.join(downloaded_pref_layout.install()))

client.run(f"cache path {created_pref}")
package_folder = client.out.strip()
assert package_folder == downloaded_pref_layout.package()
assert package_folder.endswith("p")
# Now this install will run the install() method
client.run("install --requires=dep/1.0")
assert f"Running install method in {downloaded_pref_layout.install()}" in client.out

client.run("remove * -c")
client.run("install --requires=dep/1.0 -r=default")
assert f"Running install method in {downloaded_pref_layout.install()}" in client.out