From 2d1106a17869ef8c6db16936b94a9e619b249a6d Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Thu, 16 May 2024 14:39:08 +0200 Subject: [PATCH] Drop utils --- BUILD | 4 +- MODULE.bazel | 3 - MODULE.bazel.lock | 41 +---- repositories.bzl | 1 - .../build/lib/bazel/bzlmod/RepoSpec.java | 20 +-- .../lib/skyframe/BzlmodRepoRuleFunction.java | 5 +- src/test/shell/bazel/BUILD | 5 +- src/tools/bzlmod/utils.bzl | 160 +++--------------- 8 files changed, 37 insertions(+), 202 deletions(-) diff --git a/BUILD b/BUILD index 96484633330bbe..9bedb192ddb13e 100644 --- a/BUILD +++ b/BUILD @@ -204,9 +204,7 @@ pkg_tar( write_file( name = "gen_maven_repo_name", out = "MAVEN_CANONICAL_REPO_NAME", - # TODO: Use this instead after building with Bazel 7.1.0 or later. - # content = [get_canonical_repo_name("@maven")], - content = ["rules_jvm_external~~maven~maven"], + content = [get_canonical_repo_name("@maven")], ) # The @maven repository is created by maven_install from rules_jvm_external. diff --git a/MODULE.bazel b/MODULE.bazel index b9ce01027c3027..7f598d61174e4e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -34,9 +34,6 @@ bazel_dep(name = "googletest", version = "1.14.0.bcr.1", repo_name = "com_google bazel_dep(name = "with_cfg.bzl", version = "0.2.4") bazel_dep(name = "abseil-cpp", version = "20240116.2") -# TODO(fmeum): Remove the dependency on buildozer after Bazel is built with 7.2.0. -bazel_dep(name = "buildozer", version = "7.1.2") - # TODO(pcloudy): Add remoteapis and googleapis as Bazel modules in the BCR. bazel_dep(name = "remoteapis", version = "") bazel_dep(name = "googleapis", version = "") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index ca1ed0972af680..a97b15c00f702c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -134,7 +134,7 @@ "moduleExtensions": { "//:extensions.bzl%bazel_android_deps": { "general": { - "bzlTransitiveDigest": "51/NRg2Yuex/h7TzW+J9jCXlgPIqQcGHnmKEYx6yGuY=", + "bzlTransitiveDigest": "VcSwKK3aFaFrY0hyocpNs4D/GueKeLHlKy6aIBA5zuU=", "usagesDigest": "sg97SybGLnVD/XjwZWrt/P/mpgHokHG4MDCrqAO8U4M=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -176,11 +176,6 @@ "blake3", "blake3~" ], - [ - "", - "buildozer", - "buildozer~" - ], [ "", "c-ares", @@ -280,21 +275,16 @@ "bazel_tools", "bazel_tools", "bazel_tools" - ], - [ - "buildozer~", - "buildozer_binary", - "buildozer~~buildozer_binary~buildozer_binary" ] ] } }, "//:extensions.bzl%bazel_build_deps": { "general": { - "bzlTransitiveDigest": "51/NRg2Yuex/h7TzW+J9jCXlgPIqQcGHnmKEYx6yGuY=", + "bzlTransitiveDigest": "VcSwKK3aFaFrY0hyocpNs4D/GueKeLHlKy6aIBA5zuU=", "usagesDigest": "S0vNSOLxzaSWBw2qFrqtiXI9txwZdU8Tc5Bmz/zBoJA=", "recordedFileInputs": { - "@@//MODULE.bazel": "2c56771ce2290e4701c982a8bd9df9e512b0cfa45f2ba3cf0e14457446430eb2", + "@@//MODULE.bazel": "08dbc90b1014121e86b5e6fa1f4e894d1debeb8874722ecb19700b986e3e4248", "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "4a9cf4d1d48d36a3d6e24a13094b2c32aa20be5a495d4e5b33e43db973250182" }, "recordedDirentsInputs": {}, @@ -380,7 +370,6 @@ "apple_support~", "bazel_skylib~", "blake3~", - "buildozer~", "c-ares~", "grpc~", "protobuf~", @@ -570,11 +559,6 @@ "blake3", "blake3~" ], - [ - "", - "buildozer", - "buildozer~" - ], [ "", "c-ares", @@ -674,18 +658,13 @@ "bazel_tools", "bazel_tools", "bazel_tools" - ], - [ - "buildozer~", - "buildozer_binary", - "buildozer~~buildozer_binary~buildozer_binary" ] ] } }, "//:extensions.bzl%bazel_test_deps": { "general": { - "bzlTransitiveDigest": "51/NRg2Yuex/h7TzW+J9jCXlgPIqQcGHnmKEYx6yGuY=", + "bzlTransitiveDigest": "VcSwKK3aFaFrY0hyocpNs4D/GueKeLHlKy6aIBA5zuU=", "usagesDigest": "z5vKii6UDgNY0+omarLGTmjsgS9a1wb0ayg9nBqDSxI=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -728,11 +707,6 @@ "blake3", "blake3~" ], - [ - "", - "buildozer", - "buildozer~" - ], [ "", "c-ares", @@ -832,11 +806,6 @@ "bazel_tools", "bazel_tools", "bazel_tools" - ], - [ - "buildozer~", - "buildozer_binary", - "buildozer~~buildozer_binary~buildozer_binary" ] ] } @@ -7148,4 +7117,4 @@ } } } -} \ No newline at end of file +} diff --git a/repositories.bzl b/repositories.bzl index df136d051619c1..11f27205fc2278 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -29,7 +29,6 @@ DIST_ARCHIVE_REPOS = [get_canonical_repo_name(repo) for repo in [ "apple_support", "bazel_skylib", "blake3", - "buildozer", "c-ares", "com_github_grpc_grpc", "com_google_protobuf", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java index d6ad0894c63f02..a9039ff9061d4f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java @@ -15,7 +15,6 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.auto.value.AutoValue; -import com.google.common.collect.Maps; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import javax.annotation.Nullable; @@ -52,28 +51,11 @@ public static Builder builder() { public abstract static class Builder { public abstract Builder setBzlFile(String bzlFile); - abstract String bzlFile(); - public abstract Builder setRuleClassName(String name); public abstract Builder setAttributes(AttributeValues attributes); - abstract AttributeValues attributes(); - - abstract RepoSpec autoBuild(); - - public final RepoSpec build() { - // Ensure backwards compatibility with old lockfiles that still specify the 'name' attribute. - // TODO: Remove this after both the lockfile version has been bumped and Bazel is built with - // with Bazel 7.1.0. - AttributeValues attributes = attributes(); - if (attributes.attributes().containsKey("name")) { - setAttributes( - AttributeValues.create( - Maps.filterKeys(attributes.attributes(), k -> !k.equals("name")))); - } - return autoBuild(); - } + abstract RepoSpec build(); } public boolean isNativeRepoRule() { diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java index 06ee5eade3ce9f..ed90d731f944e6 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java @@ -67,11 +67,8 @@ public final class BzlmodRepoRuleFunction implements SkyFunction { * in unambiguous canonical form and thus require no mapping, except instances read from old * lockfiles. */ - // TODO(fmeum): Make this mapping truly empty after bumping LOCK_FILE_VERSION. private static final RepositoryMapping EMPTY_MAIN_REPO_MAPPING = - RepositoryMapping.create( - ImmutableMap.of("", RepositoryName.MAIN, "bazel_tools", RepositoryName.BAZEL_TOOLS), - RepositoryName.MAIN); + RepositoryMapping.create(ImmutableMap.of(), RepositoryName.MAIN); public BzlmodRepoRuleFunction(RuleClassProvider ruleClassProvider, BlazeDirectories directories) { this.ruleClassProvider = ruleClassProvider; diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 09e0741e40b221..fa60df0dd53ebf 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -1,5 +1,6 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("//:workspace_deps.bzl", "gen_workspace_stanza") +load("//src/tools/bzlmod:utils.bzl", "get_canonical_repo_name") package(default_visibility = ["//visibility:private"]) @@ -219,9 +220,7 @@ sh_test( write_file( name = "gen_rules_java_repo_name", out = "RULES_JAVA_REPO_NAME", - # TODO: Revert to the following when building with Bazel 7.1.0. - # content = [get_canonical_repo_name("@rules_java")], - content = ["rules_java~"], + content = [get_canonical_repo_name("@rules_java")], ) sh_test( diff --git a/src/tools/bzlmod/utils.bzl b/src/tools/bzlmod/utils.bzl index c8b22a93bd26db..f8398cba4daf72 100644 --- a/src/tools/bzlmod/utils.bzl +++ b/src/tools/bzlmod/utils.bzl @@ -14,7 +14,6 @@ """Helper functions for Bzlmod build""" -load("@buildozer//:buildozer.bzl", "BUILDOZER_LABEL") load(":blazel_utils.bzl", _get_canonical_repo_name = "get_canonical_repo_name") get_canonical_repo_name = _get_canonical_repo_name @@ -51,57 +50,34 @@ def parse_http_artifacts(ctx, lockfile_path, required_repos): lockfile = json.decode(ctx.read(lockfile_path)) http_artifacts = [] found_repos = [] - if "moduleDepGraph" in lockfile: - # TODO: Remove this branch after Bazel is built with 7.2.0. - for _, module in lockfile["moduleDepGraph"].items(): - if "repoSpec" in module and module["repoSpec"]["ruleClassName"] == "http_archive": - repo_spec = module["repoSpec"] - attributes = repo_spec["attributes"] - repo_name = _module_repo_name(module) - - if repo_name not in required_repos: - continue - found_repos.append(repo_name) + for url, sha256 in lockfile["registryFileHashes"].items(): + if not url.endswith("/source.json"): + continue + segments = url.split("/") + module = { + "name": segments[-3], + "version": segments[-2], + } + repo_name = _module_repo_name(module) + if repo_name not in required_repos: + continue + found_repos.append(repo_name) - http_artifacts.append({ - "integrity": attributes["integrity"], - "url": extract_url(attributes), - }) - if "remote_patches" in attributes: - for patch, integrity in attributes["remote_patches"].items(): - http_artifacts.append({ - "integrity": integrity, - "url": patch, - }) - else: - for url, sha256 in lockfile["registryFileHashes"].items(): - if not url.endswith("/source.json"): - continue - segments = url.split("/") - module = { - "name": segments[-3], - "version": segments[-2], - } - repo_name = _module_repo_name(module) - if repo_name not in required_repos: - continue - found_repos.append(repo_name) + ctx.delete("./tempfile") + ctx.download(url, "./tempfile", executable = False, sha256 = sha256) + source_json = json.decode(ctx.read("./tempfile")) - ctx.delete("./tempfile") - ctx.download(url, "./tempfile", executable = False, sha256 = sha256) - source_json = json.decode(ctx.read("./tempfile")) + http_artifacts.append({ + "integrity": source_json["integrity"], + "url": source_json["url"], + }) + for patch, integrity in source_json.get("patches", {}).items(): http_artifacts.append({ - "integrity": source_json["integrity"], - "url": source_json["url"], + "integrity": integrity, + "url": url.rsplit("/", 1)[0] + "/patches/" + patch, }) - for patch, integrity in source_json.get("patches", {}).items(): - http_artifacts.append({ - "integrity": integrity, - "url": url.rsplit("/", 1)[0] + "/patches/" + patch, - }) - for extension_id, extension_entry in lockfile["moduleExtensions"].items(): if extension_id.startswith("@@"): # @@rules_foo~//:extensions.bzl%foo --> rules_foo~ @@ -154,66 +130,10 @@ def parse_registry_files(ctx, lockfile_path, module_files): """ lockfile = json.decode(ctx.read(lockfile_path)) registry_file_hashes = lockfile.get("registryFileHashes", {}) - if registry_file_hashes: - return [ - {"sha256": sha256, "url": url} - for url, sha256 in registry_file_hashes.items() - ] - - # TODO: Remove the following code after Bazel is built with 7.2.0. - registry_files = ["https://bcr.bazel.build/bazel_registry.json"] - - # 1. Collect all source.json files of selected module versions. - for module in lockfile["moduleDepGraph"].values(): - if module["version"]: - registry_files.append(BCR_URL_SCHEME.format( - name = module["name"], - version = module["version"], - file = "source.json", - )) - - # 2. Download registry files to compute their hashes. - registry_file_artifacts = [] - downloads = { - url: ctx.download(url, "./tempdir/{}".format(i), executable = False, block = False) - for i, url in enumerate(registry_files) - } - for url, download in downloads.items(): - hash = download.wait() - registry_file_artifacts.append({"url": url, "sha256": hash.sha256}) - - # 3. Perform module resolution in Starlark to get the MODULE.bazel file URLs - # of all module versions relevant during resolution. The lockfile only - # contains the selected module versions. - module_file_stack = [ctx.path(module_file) for module_file in module_files] - seen_deps = {} - for _ in range(1000000): - if not module_file_stack: - break - bazel_deps = _extract_bazel_deps(ctx, module_file_stack.pop()) - downloads = {} - for dep in bazel_deps: - if dep in seen_deps: - continue - url = BCR_URL_SCHEME.format( - name = dep.name, - version = dep.version, - file = "MODULE.bazel", - ) - path = ctx.path("./tempdir/modules/{name}/{version}/MODULE.bazel".format( - name = dep.name, - version = dep.version, - )) - module_file_stack.append(path) - seen_deps[dep] = None - downloads[url] = ctx.download(url, path, executable = False, block = False) - - for url, download in downloads.items(): - hash = download.wait() - registry_file_artifacts.append({"url": url, "sha256": hash.sha256}) - - ctx.delete("./tempdir") - return registry_file_artifacts + return [ + {"sha256": sha256, "url": url} + for url, sha256 in registry_file_hashes.items() + ] def parse_bazel_module_repos(ctx, lockfile_path): """Parse repo names of http_archive backed Bazel modules from the given lockfile. @@ -248,30 +168,4 @@ def _module_repo_name(module): if module_name in _WELL_KNOWN_MODULES: return module_name - # TODO(pcloudy): Simplify the following logic after we upgrade to 7.1 - if get_canonical_repo_name("rules_cc").endswith("~"): - return "{}~".format(module_name) - - return "{}~{}".format(module_name, module["version"]) - -def _extract_bazel_deps(ctx, module_file): - buildozer = ctx.path(BUILDOZER_LABEL) - temp_path = "tempdir/buildozer/MODULE.bazel" - ctx.delete(temp_path) - ctx.symlink(module_file, temp_path) - result = ctx.execute([buildozer, "print name version dev_dependency", temp_path + ":%bazel_dep"]) - if result.return_code != 0: - fail("Failed to extract bazel_dep from {}:\n{}".format(module_file, result.stderr)) - deps = [] - for line in result.stdout.splitlines(): - if " " in line: - # The dep doesn't have a version specified, which is only valid in - # the root module. Ignore it. - continue - if line.endswith(" True"): - # The dep is a dev_dependency, ignore it. - continue - name, version, _ = line.split(" ") - deps.append(struct(name = name, version = version)) - - return deps + return "{}~".format(module_name)