From af65b2c9093f10698a94ba9d027efdb732bc2f13 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 7 Mar 2024 12:23:13 -0500 Subject: [PATCH] cargo: Load Cargo.lock Cargo.lock is essentially identical to subprojects/*.wrap files. When a (sub)project has a Cargo.lock file this allows automatic fallback for its cargo dependencies. --- .../markdown/Wrap-dependency-system-manual.md | 4 ++ docs/markdown/snippets/cargo_lock.md | 6 +++ mesonbuild/cargo/__init__.py | 5 +- mesonbuild/cargo/interpreter.py | 46 +++++++++++++++++- mesonbuild/cargo/manifest.py | 17 +++++++ mesonbuild/wrap/wrap.py | 45 +++++++++-------- run_unittests.py | 2 +- test cases/rust/25 cargo lock/Cargo.lock | 7 +++ test cases/rust/25 cargo lock/meson.build | 3 ++ .../subprojects/packagecache/bar-0.1.tar.gz | Bin 0 -> 288 bytes unittests/cargotests.py | 36 +++++++++++++- 11 files changed, 147 insertions(+), 24 deletions(-) create mode 100644 docs/markdown/snippets/cargo_lock.md create mode 100644 test cases/rust/25 cargo lock/Cargo.lock create mode 100644 test cases/rust/25 cargo lock/meson.build create mode 100644 test cases/rust/25 cargo lock/subprojects/packagecache/bar-0.1.tar.gz diff --git a/docs/markdown/Wrap-dependency-system-manual.md b/docs/markdown/Wrap-dependency-system-manual.md index 5f0b473e7a2b..3983d28771e9 100644 --- a/docs/markdown/Wrap-dependency-system-manual.md +++ b/docs/markdown/Wrap-dependency-system-manual.md @@ -377,6 +377,10 @@ Some naming conventions need to be respected: - The `extra_deps` variable is pre-defined and can be used to add extra dependencies. This is typically used as `extra_deps += dependency('foo')`. +Since *1.5.0* Cargo wraps can also be provided with `Cargo.lock` file at the root +of (sub)project source tree. Meson will automatically load that file and convert +it into a serie of wraps definitions. + ## Using wrapped projects Wraps provide a convenient way of obtaining a project into your diff --git a/docs/markdown/snippets/cargo_lock.md b/docs/markdown/snippets/cargo_lock.md new file mode 100644 index 000000000000..e38c5ed35340 --- /dev/null +++ b/docs/markdown/snippets/cargo_lock.md @@ -0,0 +1,6 @@ +## Added support `Cargo.lock` file + +When a (sub)project has a `Cargo.lock` file at its root, it is loaded to provide +an automatic fallback for dependencies it defines, fetching code from +https://crates.io or git. This is identical as providing `subprojects/*.wrap`, +see [cargo wraps](Wrap-dependency-system-manual.md#cargo-wraps) dependency naming convention. diff --git a/mesonbuild/cargo/__init__.py b/mesonbuild/cargo/__init__.py index 0007b9d6469b..10cb0be103c0 100644 --- a/mesonbuild/cargo/__init__.py +++ b/mesonbuild/cargo/__init__.py @@ -1,5 +1,6 @@ __all__ = [ - 'interpret' + 'interpret', + 'load_wraps', ] -from .interpreter import interpret +from .interpreter import interpret, load_wraps diff --git a/mesonbuild/cargo/interpreter.py b/mesonbuild/cargo/interpreter.py index f1ed23953850..2eb741746f4f 100644 --- a/mesonbuild/cargo/interpreter.py +++ b/mesonbuild/cargo/interpreter.py @@ -18,12 +18,14 @@ import os import shutil import collections +import urllib.parse import typing as T from . import builder from . import version from ..mesonlib import MesonException, Popen_safe, OptionKey -from .. import coredata +from .. import coredata, mlog +from ..wrap.wrap import PackageDefinition if T.TYPE_CHECKING: from types import ModuleType @@ -731,3 +733,45 @@ def interpret(subp_name: str, subdir: str, env: Environment) -> T.Tuple[mparser. ast.extend(_create_lib(cargo, build, crate_type)) return build.block(ast), options + + +def load_wraps(source_dir: str, subproject_dir: str) -> T.List[PackageDefinition]: + """ Convert Cargo.lock into a list of wraps """ + + wraps: T.List[PackageDefinition] = [] + filename = os.path.join(source_dir, 'Cargo.lock') + if os.path.exists(filename): + cargolock = T.cast('manifest.CargoLock', load_toml(filename)) + for package in cargolock['package']: + name = package['name'] + version = package['version'] + subp_name = _dependency_name(name, _version_to_api(version)) + source = package.get('source') + if source is None: + # This is project's package, or one of its workspace members. + pass + elif source == 'registry+https://github.com/rust-lang/crates.io-index': + url = f'https://crates.io/api/v1/crates/{name}/{version}/download' + directory = f'{name}-{version}' + wraps.append(PackageDefinition.from_values(subp_name, subproject_dir, 'file', { + 'directory': directory, + 'source_url': url, + 'source_filename': f'{directory}.tar.gz', + 'source_hash': package['checksum'], + 'method': 'cargo', + })) + elif source.startswith('git+'): + parts = urllib.parse.urlparse(source[4:]) + query = urllib.parse.parse_qs(parts.query) + branch = query['branch'][0] if 'branch' in query else '' + revision = parts.fragment or branch + url = urllib.parse.urlunparse(parts._replace(params='', query='', fragment='')) + wraps.append(PackageDefinition.from_values(subp_name, subproject_dir, 'git', { + 'directory': name, + 'url': url, + 'revision': revision, + 'method': 'cargo', + })) + else: + mlog.warning(f'Unsupported source URL in {filename}: {source}') + return wraps diff --git a/mesonbuild/cargo/manifest.py b/mesonbuild/cargo/manifest.py index e6192d03cd98..183d91efd44f 100644 --- a/mesonbuild/cargo/manifest.py +++ b/mesonbuild/cargo/manifest.py @@ -225,3 +225,20 @@ class VirtualManifest(TypedDict): """ workspace: Workspace + +class CargoLockPackage(TypedDict, total=False): + + """A description of a package in the Cargo.lock file format.""" + + name: str + version: str + source: str + checksum: str + + +class CargoLock(TypedDict, total=False): + + """A description of the Cargo.lock file format.""" + + version: str + package: T.List[CargoLockPackage] diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index f853fe67d175..8b5be878ba58 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -324,25 +324,32 @@ def load_netrc(self) -> None: mlog.warning(f'failed to process netrc file: {e}.', fatal=False) def load_wraps(self) -> None: - if not os.path.isdir(self.subdir_root): - return - root, dirs, files = next(os.walk(self.subdir_root)) - ignore_dirs = {'packagecache', 'packagefiles'} - for i in files: - if not i.endswith('.wrap'): - continue - fname = os.path.join(self.subdir_root, i) - wrap = PackageDefinition.from_wrap_file(fname, self.subproject) - self.wraps[wrap.name] = wrap - ignore_dirs |= {wrap.directory, wrap.name} - # Add dummy package definition for directories not associated with a wrap file. - for i in dirs: - if i in ignore_dirs: - continue - fname = os.path.join(self.subdir_root, i) - wrap = PackageDefinition.from_directory(fname) - self.wraps[wrap.name] = wrap - + # Load Cargo.lock at the root of source tree + source_dir = os.path.dirname(self.subdir_root) + if os.path.exists(os.path.join(source_dir, 'Cargo.lock')): + from .. import cargo + for wrap in cargo.load_wraps(source_dir, self.subdir_root): + self.wraps[wrap.name] = wrap + # Load subprojects/*.wrap + if os.path.isdir(self.subdir_root): + root, dirs, files = next(os.walk(self.subdir_root)) + for i in files: + if not i.endswith('.wrap'): + continue + fname = os.path.join(self.subdir_root, i) + wrap = PackageDefinition.from_wrap_file(fname, self.subproject) + self.wraps[wrap.name] = wrap + # Add dummy package definition for directories not associated with a wrap file. + ignore_dirs = {'packagecache', 'packagefiles'} + for wrap in self.wraps.values(): + ignore_dirs |= {wrap.directory, wrap.name} + for i in dirs: + if i in ignore_dirs: + continue + fname = os.path.join(self.subdir_root, i) + wrap = PackageDefinition.from_directory(fname) + self.wraps[wrap.name] = wrap + # Add provided deps and programs into our lookup tables for wrap in self.wraps.values(): self.add_wrap(wrap) diff --git a/run_unittests.py b/run_unittests.py index 33b0e0988753..84edb34ccd48 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -25,7 +25,7 @@ import mesonbuild.modules.pkgconfig from unittests.allplatformstests import AllPlatformTests -from unittests.cargotests import CargoVersionTest, CargoCfgTest +from unittests.cargotests import CargoVersionTest, CargoCfgTest, CargoLockTest from unittests.darwintests import DarwinTests from unittests.failuretests import FailureTests from unittests.linuxcrosstests import LinuxCrossArmTests, LinuxCrossMingwTests diff --git a/test cases/rust/25 cargo lock/Cargo.lock b/test cases/rust/25 cargo lock/Cargo.lock new file mode 100644 index 000000000000..9bc98149bb3f --- /dev/null +++ b/test cases/rust/25 cargo lock/Cargo.lock @@ -0,0 +1,7 @@ +version = 3 + +[[package]] +name = "bar" +version = "0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2f34e570dcd5f9fe32e6863ee16ee73a356d3b77bce0d8c78501b8bc81a860" diff --git a/test cases/rust/25 cargo lock/meson.build b/test cases/rust/25 cargo lock/meson.build new file mode 100644 index 000000000000..b359f7bb0aa7 --- /dev/null +++ b/test cases/rust/25 cargo lock/meson.build @@ -0,0 +1,3 @@ +project('cargo lock') + +dependency('bar-0.1-rs') diff --git a/test cases/rust/25 cargo lock/subprojects/packagecache/bar-0.1.tar.gz b/test cases/rust/25 cargo lock/subprojects/packagecache/bar-0.1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f4c2ec6725a62f65800ad2128d3567ad355d534e GIT binary patch literal 288 zcmV+*0pI@pH+ooF000E$*0e?f03iVu0001VFXf})5B~t)T>vo{N~ZzB@8=`!?Itm7 zu@4Ldiz)|;h(iPBVpT6cc*BFb|CH}oZpL--w7kPIV{xsoTeO4_4OsmqFwFsa37Z9a z46y)>KzUKcC8{->*jl`R)BzH$68O@7@uq*BEP2)HphXtU8T%oci3@zZDIF-bSw1Mu z7c57i{{kaL>#%YL?5?kN!JyhmAzjD$a&|AHG+m!785+(i0A|y!u?=9lCj+UArIDDJ zc5h_%{~}_0eCqk-ZyHCT{}Pc#RDuTZ`UD-+3Qv;G*(v3ap`aMma@^=}rO6b}9J{`o m0002vnI&H)zm`q_0s8@fAOHYt None: with self.subTest(): value = cfg.ir_to_meson(cfg.parse(iter(cfg.lexer(data))), build) self.assertEqual(value, expected) + +class CargoLockTest(unittest.TestCase): + def test_cargo_lock(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + with open(os.path.join(tmpdir, 'Cargo.lock'), 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + version = 3 + [[package]] + name = "foo" + version = "0.1" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" + [[package]] + name = "bar" + version = "0.1" + source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=0.19#23c5599424cc75ec66618891c915d9f490f6e4c2" + ''')) + wraps = load_wraps(tmpdir, 'subprojects') + self.assertEqual(len(wraps), 2) + self.assertEqual(wraps[0].name, 'foo-0.1-rs') + self.assertEqual(wraps[0].directory, 'foo-0.1') + self.assertEqual(wraps[0].type, 'file') + self.assertEqual(wraps[0].get('method'), 'cargo') + self.assertEqual(wraps[0].get('source_url'), 'https://crates.io/api/v1/crates/foo/0.1/download') + self.assertEqual(wraps[0].get('source_hash'), '8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb') + self.assertEqual(wraps[1].name, 'bar-0.1-rs') + self.assertEqual(wraps[1].directory, 'bar') + self.assertEqual(wraps[1].type, 'git') + self.assertEqual(wraps[1].get('method'), 'cargo') + self.assertEqual(wraps[1].get('url'), 'https://github.com/gtk-rs/gtk-rs-core') + self.assertEqual(wraps[1].get('revision'), '23c5599424cc75ec66618891c915d9f490f6e4c2')