Skip to content

Commit

Permalink
Add bazel mod tidy
Browse files Browse the repository at this point in the history
Implements https://docs.google.com/document/d/13LbK_1WhA4la0eH7yISjnMvXs2cKFXD-adKPu0i0RK0/edit

RELNOTES: The new `bazel mod tidy` subcommand automatically updates `use_repo` calls in the `MODULE.bazel` file for extensions that use `module_ctx.extension_metadata`.

Closes #20483.

PiperOrigin-RevId: 605021386
Change-Id: Idb1f22c51e126328b9efd6a5a6d6f89d77b9308d
  • Loading branch information
fmeum committed Feb 8, 2024
1 parent a91fce1 commit 754e694
Show file tree
Hide file tree
Showing 26 changed files with 1,130 additions and 208 deletions.
5 changes: 5 additions & 0 deletions src/MODULE.tools
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ bazel_dep(name = "rules_license", version = "0.0.3")
bazel_dep(name = "rules_proto", version = "4.0.0")
bazel_dep(name = "rules_python", version = "0.22.0")

bazel_dep(name = "buildozer", version = "6.4.0.2")
bazel_dep(name = "platforms", version = "0.0.7")
bazel_dep(name = "protobuf", version = "3.19.6", repo_name = "com_google_protobuf")
bazel_dep(name = "zlib", version = "1.3")
Expand Down Expand Up @@ -42,5 +43,9 @@ use_repo(remote_coverage_tools_extension, "remote_coverage_tools")
remote_android_extensions = use_extension("//tools/android:android_extensions.bzl", "remote_android_tools_extensions")
use_repo(remote_android_extensions, "android_gmaven_r8", "android_tools")

# Used by bazel mod tidy (see BazelModTidyFunction).
buildozer_binary = use_extension("@buildozer//:buildozer_binary.bzl", "buildozer_binary")
use_repo(buildozer_binary, "buildozer_binary")

# Platforms used by transitions in builtins
bazel_dep(name = "apple_support", version = "1.5.0", repo_name = "build_bazel_apple_support")
3 changes: 1 addition & 2 deletions src/main/java/com/google/devtools/build/lib/bazel/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,11 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/authandtls",
"//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper",
"//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection_impl",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:registry",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:tidy_impl",
"//src/main/java/com/google/devtools/build/lib/bazel/commands",
"//src/main/java/com/google/devtools/build/lib/bazel/repository",
"//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,11 @@
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
import com.google.devtools.build.lib.bazel.bzlmod.AttributeValues;
import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction;
import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllFunction;
import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModTidyFunction;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorFunction;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction;
import com.google.devtools.build.lib.bazel.bzlmod.LocalPathOverride;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionRepoMappingEntriesFunction;
Expand All @@ -48,7 +47,6 @@
import com.google.devtools.build.lib.bazel.bzlmod.NonRegistryOverride;
import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactory;
import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactoryImpl;
import com.google.devtools.build.lib.bazel.bzlmod.RepoSpec;
import com.google.devtools.build.lib.bazel.bzlmod.RepoSpecFunction;
import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionEvalFunction;
import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionUsagesFunction;
Expand Down Expand Up @@ -245,36 +243,7 @@ public void workspaceInit(
new SingleExtensionEvalFunction(directories, clientEnvironmentSupplier, downloadManager);

if (builtinModules == null) {
builtinModules =
ImmutableMap.of(
// @bazel_tools is a special repo that we pull from the extracted install dir.
"bazel_tools",
LocalPathOverride.create(
directories.getEmbeddedBinariesRoot().getChild("embedded_tools").getPathString()),
// @local_config_platform is currently generated by the native repo rule
// local_config_platform
// It has to be a special repo for now because:
// - It's embedded in local_config_platform.WORKSPACE and depended on by many
// toolchains.
// - The canonical name "local_config_platform" is hardcoded in Bazel code.
// See {@link PlatformOptions}
"local_config_platform",
new NonRegistryOverride() {
@Override
public RepoSpec getRepoSpec() {
return RepoSpec.builder()
.setRuleClassName("local_config_platform")
.setAttributes(AttributeValues.create(ImmutableMap.of()))
.build();
}

@Override
public ResolutionReason getResolutionReason() {
// NOTE: It is not exactly a LOCAL_PATH_OVERRIDE, but there is no inspection
// ResolutionReason for builtin modules
return ResolutionReason.LOCAL_PATH_OVERRIDE;
}
});
builtinModules = ModuleFileFunction.getBuiltinModules(directories.getEmbeddedBinariesRoot());
}

builder
Expand All @@ -286,6 +255,7 @@ public ResolutionReason getResolutionReason() {
.addSkyFunction(
SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace()))
.addSkyFunction(SkyFunctions.BAZEL_FETCH_ALL, new BazelFetchAllFunction())
.addSkyFunction(SkyFunctions.BAZEL_MOD_TIDY, new BazelModTidyFunction())
.addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction())
.addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction())
.addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction)
Expand Down
57 changes: 57 additions & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ java_library(
deps = [
":common",
":exception",
":inspection",
":module_extension",
":module_extension_metadata",
":registry",
Expand Down Expand Up @@ -271,6 +272,49 @@ java_library(
],
)

java_library(
name = "tidy",
srcs = [
"BazelModTidyValue.java",
],
deps = [
":resolution",
"//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
"//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:serialization-constant",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
"//src/main/java/net/starlark/java/eval",
"//third_party:auto_value",
"//third_party:guava",
],
)

java_library(
name = "tidy_impl",
srcs = [
"BazelModTidyFunction.java",
],
deps = [
":common",
":exception",
":resolution",
":resolution_impl",
":tidy",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function",
"//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/skyframe",
"//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
"//src/main/java/net/starlark/java/eval",
"//src/main/protobuf:failure_details_java_proto",
"//third_party:guava",
"//third_party:jsr305",
],
)

java_library(
name = "inspection",
srcs = [
Expand Down Expand Up @@ -312,6 +356,7 @@ java_library(
deps = [
":common",
":module_extension",
":module_file_fixup_event",
"//src/main/java/com/google/devtools/build/docgen/annot",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/events",
Expand All @@ -324,3 +369,15 @@ java_library(
"//third_party:jsr305",
],
)

java_library(
name = "module_file_fixup_event",
srcs = [
"RootModuleFileFixupEvent.java",
],
deps = [
":module_extension",
"//src/main/java/com/google/devtools/build/lib/events",
"//third_party:guava",
],
)
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,8 @@ public static void updateLockfile(Path workspaceRoot, BazelLockFileValue updated

@Subscribe
public void bazelModuleResolved(BazelModuleResolutionEvent moduleResolutionEvent) {
// Latest event wins, which is relevant in the case of `bazel mod tidy`, where a new event is
// sent after the command has modified the module file.
this.moduleResolutionEvent = moduleResolutionEvent;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

package com.google.devtools.build.lib.bazel.bzlmod;

import static com.google.devtools.build.lib.bazel.bzlmod.InterimModule.toModule;

import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -117,4 +118,24 @@ public ImmutableList<String> getModuleAndFlagsDiff(
}
return moduleDiff.build();
}

/**
* Returns a new BazelLockFileValue in which all information about the root module has been
* replaced by the given value.
*
* <p>This operation is shallow: If the new root module has different dependencies, the dep graph
* will not be updated.
*/
public BazelLockFileValue withShallowlyReplacedRootModule(
ModuleFileValue.RootModuleFileValue value) {
ImmutableMap.Builder<ModuleKey, Module> newDepGraph = ImmutableMap.builder();
newDepGraph.putAll(getModuleDepGraph());
newDepGraph.put(
ModuleKey.ROOT,
toModule(value.getModule(), /* override= */ null, /* remoteRepoSpec= */ null));
return toBuilder()
.setModuleFileHash(value.getModuleFileHash())
.setModuleDepGraph(newDepGraph.buildKeepingLast())
.build();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// Copyright 2022 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package com.google.devtools.build.lib.bazel.bzlmod;

import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction.LOCKFILE_MODE;
import static com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction.IGNORE_DEV_DEPS;
import static com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction.MODULE_OVERRIDES;
import static com.google.devtools.build.lib.skyframe.PrecomputedValue.STARLARK_SEMANTICS;

import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.skyframe.RepositoryMappingValue;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.SkyframeLookupResult;
import javax.annotation.Nullable;
import net.starlark.java.eval.EvalException;

/**
* Computes all information required for the {@code bazel mod tidy} command. The evaluation of all
* module extensions used by the root module is triggered to, as a side effect, emit any {@link
* RootModuleFileFixupEvent}s.
*/
public class BazelModTidyFunction implements SkyFunction {

@Override
@Nullable
public SkyValue compute(SkyKey skyKey, Environment env)
throws InterruptedException, SkyFunctionException {
BazelDepGraphValue depGraphValue = (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY);
if (depGraphValue == null) {
return null;
}
RepositoryMappingValue bazelToolsRepoMapping =
(RepositoryMappingValue)
env.getValue(RepositoryMappingValue.key(RepositoryName.BAZEL_TOOLS));
if (bazelToolsRepoMapping == null) {
return null;
}
Label buildozerLabel;
try {
buildozerLabel =
Label.parseWithRepoContext(
// This label always has the ".exe" extension, even on Unix, to get a single static
// label that works on all platforms.
"@buildozer_binary//:buildozer.exe",
Label.RepoContext.of(
RepositoryName.BAZEL_TOOLS, bazelToolsRepoMapping.getRepositoryMapping()));
} catch (LabelSyntaxException e) {
throw new IllegalStateException(e);
}
RootedPath buildozer;
try {
buildozer = RepositoryFunction.getRootedPathFromLabel(buildozerLabel, env);
} catch (NeedsSkyframeRestartException e) {
return null;
} catch (EvalException e) {
throw new IllegalStateException(e);
}

ImmutableSet<SkyKey> extensionsUsedByRootModule =
depGraphValue.getExtensionUsagesTable().columnMap().get(ModuleKey.ROOT).keySet().stream()
.map(SingleExtensionEvalValue::key)
.collect(toImmutableSet());
SkyframeLookupResult result = env.getValuesAndExceptions(extensionsUsedByRootModule);
if (env.valuesMissing()) {
return null;
}
for (SkyKey extension : extensionsUsedByRootModule) {
try {
result.getOrThrow(extension, ExternalDepsException.class);
} catch (ExternalDepsException e) {
if (e.getDetailedExitCode().getFailureDetail() == null
|| !e.getDetailedExitCode()
.getFailureDetail()
.getExternalDeps()
.getCode()
.equals(FailureDetails.ExternalDeps.Code.INVALID_EXTENSION_IMPORT)) {
throw new BazelModTidyFunctionException(e, SkyFunctionException.Transience.PERSISTENT);
}
// This is an error bazel mod tidy can fix, so don't fail.
}
}

return BazelModTidyValue.create(
buildozer.asPath(),
MODULE_OVERRIDES.get(env),
IGNORE_DEV_DEPS.get(env),
LOCKFILE_MODE.get(env),
STARLARK_SEMANTICS.get(env));
}

static final class BazelModTidyFunctionException extends SkyFunctionException {

BazelModTidyFunctionException(ExternalDepsException cause, Transience transience) {
super(cause, transience);
}
}
}
Loading

0 comments on commit 754e694

Please sign in to comment.