Skip to content

Commit

Permalink
Drop utils
Browse files Browse the repository at this point in the history
  • Loading branch information
fmeum committed Jun 11, 2024
1 parent dc62a5f commit e926983
Show file tree
Hide file tree
Showing 8 changed files with 36 additions and 202 deletions.
4 changes: 1 addition & 3 deletions BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -204,9 +204,7 @@ pkg_tar(
write_file(
name = "gen_maven_repo_name",
out = "MAVEN_CANONICAL_REPO_NAME",
# TODO: Use this instead after building with Bazel 7.1.0 or later.
# content = [get_canonical_repo_name("@maven")],
content = ["rules_jvm_external~~maven~maven"],
content = [get_canonical_repo_name("@maven")],
)

# The @maven repository is created by maven_install from rules_jvm_external.
Expand Down
3 changes: 0 additions & 3 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,6 @@ bazel_dep(name = "googletest", version = "1.14.0.bcr.1", repo_name = "com_google
bazel_dep(name = "with_cfg.bzl", version = "0.2.4")
bazel_dep(name = "abseil-cpp", version = "20240116.2")

# TODO(fmeum): Remove the dependency on buildozer after Bazel is built with 7.2.0.
bazel_dep(name = "buildozer", version = "7.1.2")

# TODO(pcloudy): Add remoteapis and googleapis as Bazel modules in the BCR.
bazel_dep(name = "remoteapis", version = "")
bazel_dep(name = "googleapis", version = "")
Expand Down
41 changes: 5 additions & 36 deletions MODULE.bazel.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion repositories.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ DIST_ARCHIVE_REPOS = [get_canonical_repo_name(repo) for repo in [
"apple_support",
"bazel_skylib",
"blake3",
"buildozer",
"c-ares",
"com_github_grpc_grpc",
"com_google_protobuf",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
package com.google.devtools.build.lib.bazel.bzlmod;

import com.google.auto.value.AutoValue;
import com.google.common.collect.Maps;
import com.ryanharter.auto.value.gson.GenerateTypeAdapter;
import javax.annotation.Nullable;

Expand Down Expand Up @@ -52,28 +51,11 @@ public static Builder builder() {
public abstract static class Builder {
public abstract Builder setBzlFile(String bzlFile);

abstract String bzlFile();

public abstract Builder setRuleClassName(String name);

public abstract Builder setAttributes(AttributeValues attributes);

abstract AttributeValues attributes();

abstract RepoSpec autoBuild();

public final RepoSpec build() {
// Ensure backwards compatibility with old lockfiles that still specify the 'name' attribute.
// TODO: Remove this after both the lockfile version has been bumped and Bazel is built with
// with Bazel 7.1.0.
AttributeValues attributes = attributes();
if (attributes.attributes().containsKey("name")) {
setAttributes(
AttributeValues.create(
Maps.filterKeys(attributes.attributes(), k -> !k.equals("name"))));
}
return autoBuild();
}
abstract RepoSpec build();
}

public boolean isNativeRepoRule() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,8 @@ public final class BzlmodRepoRuleFunction implements SkyFunction {
* in unambiguous canonical form and thus require no mapping, except instances read from old
* lockfiles.
*/
// TODO(fmeum): Make this mapping truly empty after bumping LOCK_FILE_VERSION.
private static final RepositoryMapping EMPTY_MAIN_REPO_MAPPING =
RepositoryMapping.create(
ImmutableMap.of("", RepositoryName.MAIN, "bazel_tools", RepositoryName.BAZEL_TOOLS),
RepositoryName.MAIN);
RepositoryMapping.create(ImmutableMap.of(), RepositoryName.MAIN);

public BzlmodRepoRuleFunction(RuleClassProvider ruleClassProvider, BlazeDirectories directories) {
this.ruleClassProvider = ruleClassProvider;
Expand Down
4 changes: 1 addition & 3 deletions src/test/shell/bazel/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -219,9 +219,7 @@ sh_test(
write_file(
name = "gen_rules_java_repo_name",
out = "RULES_JAVA_REPO_NAME",
# TODO: Revert to the following when building with Bazel 7.1.0.
# content = [get_canonical_repo_name("@rules_java")],
content = ["rules_java~"],
content = [get_canonical_repo_name("@rules_java")],
)

sh_test(
Expand Down
160 changes: 27 additions & 133 deletions src/tools/bzlmod/utils.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

"""Helper functions for Bzlmod build"""

load("@buildozer//:buildozer.bzl", "BUILDOZER_LABEL")
load(":blazel_utils.bzl", _get_canonical_repo_name = "get_canonical_repo_name")

get_canonical_repo_name = _get_canonical_repo_name
Expand Down Expand Up @@ -51,57 +50,34 @@ def parse_http_artifacts(ctx, lockfile_path, required_repos):
lockfile = json.decode(ctx.read(lockfile_path))
http_artifacts = []
found_repos = []
if "moduleDepGraph" in lockfile:
# TODO: Remove this branch after Bazel is built with 7.2.0.
for _, module in lockfile["moduleDepGraph"].items():
if "repoSpec" in module and module["repoSpec"]["ruleClassName"] == "http_archive":
repo_spec = module["repoSpec"]
attributes = repo_spec["attributes"]
repo_name = _module_repo_name(module)

if repo_name not in required_repos:
continue
found_repos.append(repo_name)
for url, sha256 in lockfile["registryFileHashes"].items():
if not url.endswith("/source.json"):
continue
segments = url.split("/")
module = {
"name": segments[-3],
"version": segments[-2],
}
repo_name = _module_repo_name(module)
if repo_name not in required_repos:
continue
found_repos.append(repo_name)

http_artifacts.append({
"integrity": attributes["integrity"],
"url": extract_url(attributes),
})
if "remote_patches" in attributes:
for patch, integrity in attributes["remote_patches"].items():
http_artifacts.append({
"integrity": integrity,
"url": patch,
})
else:
for url, sha256 in lockfile["registryFileHashes"].items():
if not url.endswith("/source.json"):
continue
segments = url.split("/")
module = {
"name": segments[-3],
"version": segments[-2],
}
repo_name = _module_repo_name(module)
if repo_name not in required_repos:
continue
found_repos.append(repo_name)
ctx.delete("./tempfile")
ctx.download(url, "./tempfile", executable = False, sha256 = sha256)
source_json = json.decode(ctx.read("./tempfile"))

ctx.delete("./tempfile")
ctx.download(url, "./tempfile", executable = False, sha256 = sha256)
source_json = json.decode(ctx.read("./tempfile"))
http_artifacts.append({
"integrity": source_json["integrity"],
"url": source_json["url"],
})

for patch, integrity in source_json.get("patches", {}).items():
http_artifacts.append({
"integrity": source_json["integrity"],
"url": source_json["url"],
"integrity": integrity,
"url": url.rsplit("/", 1)[0] + "/patches/" + patch,
})

for patch, integrity in source_json.get("patches", {}).items():
http_artifacts.append({
"integrity": integrity,
"url": url.rsplit("/", 1)[0] + "/patches/" + patch,
})

for extension_id, extension_entry in lockfile["moduleExtensions"].items():
if extension_id.startswith("@@"):
# @@rules_foo~//:extensions.bzl%foo --> rules_foo~
Expand Down Expand Up @@ -154,66 +130,10 @@ def parse_registry_files(ctx, lockfile_path, module_files):
"""
lockfile = json.decode(ctx.read(lockfile_path))
registry_file_hashes = lockfile.get("registryFileHashes", {})
if registry_file_hashes:
return [
{"sha256": sha256, "url": url}
for url, sha256 in registry_file_hashes.items()
]

# TODO: Remove the following code after Bazel is built with 7.2.0.
registry_files = ["https://bcr.bazel.build/bazel_registry.json"]

# 1. Collect all source.json files of selected module versions.
for module in lockfile["moduleDepGraph"].values():
if module["version"]:
registry_files.append(BCR_URL_SCHEME.format(
name = module["name"],
version = module["version"],
file = "source.json",
))

# 2. Download registry files to compute their hashes.
registry_file_artifacts = []
downloads = {
url: ctx.download(url, "./tempdir/{}".format(i), executable = False, block = False)
for i, url in enumerate(registry_files)
}
for url, download in downloads.items():
hash = download.wait()
registry_file_artifacts.append({"url": url, "sha256": hash.sha256})

# 3. Perform module resolution in Starlark to get the MODULE.bazel file URLs
# of all module versions relevant during resolution. The lockfile only
# contains the selected module versions.
module_file_stack = [ctx.path(module_file) for module_file in module_files]
seen_deps = {}
for _ in range(1000000):
if not module_file_stack:
break
bazel_deps = _extract_bazel_deps(ctx, module_file_stack.pop())
downloads = {}
for dep in bazel_deps:
if dep in seen_deps:
continue
url = BCR_URL_SCHEME.format(
name = dep.name,
version = dep.version,
file = "MODULE.bazel",
)
path = ctx.path("./tempdir/modules/{name}/{version}/MODULE.bazel".format(
name = dep.name,
version = dep.version,
))
module_file_stack.append(path)
seen_deps[dep] = None
downloads[url] = ctx.download(url, path, executable = False, block = False)

for url, download in downloads.items():
hash = download.wait()
registry_file_artifacts.append({"url": url, "sha256": hash.sha256})

ctx.delete("./tempdir")
return registry_file_artifacts
return [
{"sha256": sha256, "url": url}
for url, sha256 in registry_file_hashes.items()
]

def parse_bazel_module_repos(ctx, lockfile_path):
"""Parse repo names of http_archive backed Bazel modules from the given lockfile.
Expand Down Expand Up @@ -248,30 +168,4 @@ def _module_repo_name(module):
if module_name in _WELL_KNOWN_MODULES:
return module_name

# TODO(pcloudy): Simplify the following logic after we upgrade to 7.1
if get_canonical_repo_name("rules_cc").endswith("~"):
return "{}~".format(module_name)

return "{}~{}".format(module_name, module["version"])

def _extract_bazel_deps(ctx, module_file):
buildozer = ctx.path(BUILDOZER_LABEL)
temp_path = "tempdir/buildozer/MODULE.bazel"
ctx.delete(temp_path)
ctx.symlink(module_file, temp_path)
result = ctx.execute([buildozer, "print name version dev_dependency", temp_path + ":%bazel_dep"])
if result.return_code != 0:
fail("Failed to extract bazel_dep from {}:\n{}".format(module_file, result.stderr))
deps = []
for line in result.stdout.splitlines():
if " " in line:
# The dep doesn't have a version specified, which is only valid in
# the root module. Ignore it.
continue
if line.endswith(" True"):
# The dep is a dev_dependency, ignore it.
continue
name, version, _ = line.split(" ")
deps.append(struct(name = name, version = version))

return deps
return "{}~".format(module_name)

0 comments on commit e926983

Please sign in to comment.