[7.1.0] Add bazel mod tidy

src/MODULE.tools

@@ -10,6 +10,7 @@ bazel_dep(name = "rules_license", version = "0.0.3")
 bazel_dep(name = "rules_proto", version = "4.0.0")
 bazel_dep(name = "rules_python", version = "0.22.0")
 
+bazel_dep(name = "buildozer", version = "6.4.0.2")
 bazel_dep(name = "platforms", version = "0.0.7")
 bazel_dep(name = "protobuf", version = "3.19.6", repo_name = "com_google_protobuf")
 bazel_dep(name = "zlib", version = "1.3")
@@ -42,5 +43,9 @@ use_repo(remote_coverage_tools_extension, "remote_coverage_tools")
 remote_android_extensions = use_extension("//tools/android:android_extensions.bzl", "remote_android_tools_extensions")
 use_repo(remote_android_extensions, "android_gmaven_r8", "android_tools")
 
+# Used by bazel mod tidy (see BazelModTidyFunction).
+buildozer_binary = use_extension("@buildozer//:buildozer_binary.bzl", "buildozer_binary")
+use_repo(buildozer_binary, "buildozer_binary")
+
 # Platforms used by transitions in builtins
 bazel_dep(name = "apple_support", version = "1.5.0", repo_name = "build_bazel_apple_support")

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -29,12 +29,11 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/authandtls",
         "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper",
         "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
-        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common",
-        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection_impl",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:registry",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl",
+        "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:tidy_impl",
         "//src/main/java/com/google/devtools/build/lib/bazel/commands",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -34,12 +34,11 @@
 import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
 import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider;
 import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
-import com.google.devtools.build.lib.bazel.bzlmod.AttributeValues;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelFetchAllFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction;
+import com.google.devtools.build.lib.bazel.bzlmod.BazelModTidyFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorFunction;
-import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.LocalPathOverride;
 import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionRepoMappingEntriesFunction;
@@ -48,7 +47,6 @@
 import com.google.devtools.build.lib.bazel.bzlmod.NonRegistryOverride;
 import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactory;
 import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactoryImpl;
-import com.google.devtools.build.lib.bazel.bzlmod.RepoSpec;
 import com.google.devtools.build.lib.bazel.bzlmod.RepoSpecFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionEvalFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionUsagesFunction;
@@ -245,36 +243,7 @@ public void workspaceInit(
         new SingleExtensionEvalFunction(directories, clientEnvironmentSupplier, downloadManager);
 
     if (builtinModules == null) {
-      builtinModules =
-          ImmutableMap.of(
-              // @bazel_tools is a special repo that we pull from the extracted install dir.
-              "bazel_tools",
-              LocalPathOverride.create(
-                  directories.getEmbeddedBinariesRoot().getChild("embedded_tools").getPathString()),
-              // @local_config_platform is currently generated by the native repo rule
-              // local_config_platform
-              // It has to be a special repo for now because:
-              //   - It's embedded in local_config_platform.WORKSPACE and depended on by many
-              // toolchains.
-              //   - The canonical name "local_config_platform" is hardcoded in Bazel code.
-              //     See {@link PlatformOptions}
-              "local_config_platform",
-              new NonRegistryOverride() {
-                @Override
-                public RepoSpec getRepoSpec() {
-                  return RepoSpec.builder()
-                      .setRuleClassName("local_config_platform")
-                      .setAttributes(AttributeValues.create(ImmutableMap.of()))
-                      .build();
-                }
-
-                @Override
-                public ResolutionReason getResolutionReason() {
-                  // NOTE: It is not exactly a LOCAL_PATH_OVERRIDE, but there is no inspection
-                  // ResolutionReason for builtin modules
-                  return ResolutionReason.LOCAL_PATH_OVERRIDE;
-                }
-              });
+      builtinModules = ModuleFileFunction.getBuiltinModules(directories.getEmbeddedBinariesRoot());
     }
 
     builder
@@ -286,6 +255,7 @@ public ResolutionReason getResolutionReason() {
         .addSkyFunction(
             SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace()))
         .addSkyFunction(SkyFunctions.BAZEL_FETCH_ALL, new BazelFetchAllFunction())
+        .addSkyFunction(SkyFunctions.BAZEL_MOD_TIDY, new BazelModTidyFunction())
         .addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction())
         .addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction())
         .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction)

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD

@@ -196,6 +196,7 @@ java_library(
     deps = [
         ":common",
         ":exception",
+        ":inspection",
         ":module_extension",
         ":module_extension_metadata",
         ":registry",
@@ -271,6 +272,49 @@ java_library(
     ],
 )
 
+java_library(
+    name = "tidy",
+    srcs = [
+        "BazelModTidyValue.java",
+    ],
+    deps = [
+        ":resolution",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions",
+        "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:serialization-constant",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+        "//src/main/java/net/starlark/java/eval",
+        "//third_party:auto_value",
+        "//third_party:guava",
+    ],
+)
+
+java_library(
+    name = "tidy_impl",
+    srcs = [
+        "BazelModTidyFunction.java",
+    ],
+    deps = [
+        ":common",
+        ":exception",
+        ":resolution",
+        ":resolution_impl",
+        ":tidy",
+        "//src/main/java/com/google/devtools/build/lib/cmdline",
+        "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/skyframe",
+        "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+        "//src/main/java/net/starlark/java/eval",
+        "//src/main/protobuf:failure_details_java_proto",
+        "//third_party:guava",
+        "//third_party:jsr305",
+    ],
+)
+
 java_library(
     name = "inspection",
     srcs = [
@@ -312,6 +356,7 @@ java_library(
     deps = [
         ":common",
         ":module_extension",
+        ":module_file_fixup_event",
         "//src/main/java/com/google/devtools/build/docgen/annot",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/events",
@@ -324,3 +369,15 @@ java_library(
         "//third_party:jsr305",
     ],
 )
+
+java_library(
+    name = "module_file_fixup_event",
+    srcs = [
+        "RootModuleFileFixupEvent.java",
+    ],
+    deps = [
+        ":module_extension",
+        "//src/main/java/com/google/devtools/build/lib/events",
+        "//third_party:guava",
+    ],
+)

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java

@@ -213,6 +213,8 @@ public static void updateLockfile(Path workspaceRoot, BazelLockFileValue updated
 
   @Subscribe
   public void bazelModuleResolved(BazelModuleResolutionEvent moduleResolutionEvent) {
+    // Latest event wins, which is relevant in the case of `bazel mod tidy`, where a new event is
+    // sent after the command has modified the module file.
     this.moduleResolutionEvent = moduleResolutionEvent;
   }
 

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java

@@ -15,6 +15,7 @@
 
 package com.google.devtools.build.lib.bazel.bzlmod;
 
+import static com.google.devtools.build.lib.bazel.bzlmod.InterimModule.toModule;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
@@ -117,4 +118,24 @@ public ImmutableList<String> getModuleAndFlagsDiff(
     }
     return moduleDiff.build();
   }
+
+  /**
+   * Returns a new BazelLockFileValue in which all information about the root module has been
+   * replaced by the given value.
+   *
+   * <p>This operation is shallow: If the new root module has different dependencies, the dep graph
+   * will not be updated.
+   */
+  public BazelLockFileValue withShallowlyReplacedRootModule(
+      ModuleFileValue.RootModuleFileValue value) {
+    ImmutableMap.Builder<ModuleKey, Module> newDepGraph = ImmutableMap.builder();
+    newDepGraph.putAll(getModuleDepGraph());
+    newDepGraph.put(
+        ModuleKey.ROOT,
+        toModule(value.getModule(), /* override= */ null, /* remoteRepoSpec= */ null));
+    return toBuilder()
+        .setModuleFileHash(value.getModuleFileHash())
+        .setModuleDepGraph(newDepGraph.buildKeepingLast())
+        .build();
+  }
 }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyFunction.java

@@ -0,0 +1,121 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package com.google.devtools.build.lib.bazel.bzlmod;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction.LOCKFILE_MODE;
+import static com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction.IGNORE_DEV_DEPS;
+import static com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction.MODULE_OVERRIDES;
+import static com.google.devtools.build.lib.skyframe.PrecomputedValue.STARLARK_SEMANTICS;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
+import com.google.devtools.build.lib.cmdline.RepositoryName;
+import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException;
+import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
+import com.google.devtools.build.lib.server.FailureDetails;
+import com.google.devtools.build.lib.skyframe.RepositoryMappingValue;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.SkyframeLookupResult;
+import javax.annotation.Nullable;
+import net.starlark.java.eval.EvalException;
+
+/**
+ * Computes all information required for the {@code bazel mod tidy} command. The evaluation of all
+ * module extensions used by the root module is triggered to, as a side effect, emit any {@link
+ * RootModuleFileFixupEvent}s.
+ */
+public class BazelModTidyFunction implements SkyFunction {
+
+  @Override
+  @Nullable
+  public SkyValue compute(SkyKey skyKey, Environment env)
+      throws InterruptedException, SkyFunctionException {
+    BazelDepGraphValue depGraphValue = (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY);
+    if (depGraphValue == null) {
+      return null;
+    }
+    RepositoryMappingValue bazelToolsRepoMapping =
+        (RepositoryMappingValue)
+            env.getValue(RepositoryMappingValue.key(RepositoryName.BAZEL_TOOLS));
+    if (bazelToolsRepoMapping == null) {
+      return null;
+    }
+    Label buildozerLabel;
+    try {
+      buildozerLabel =
+          Label.parseWithRepoContext(
+              // This label always has the ".exe" extension, even on Unix, to get a single static
+              // label that works on all platforms.
+              "@buildozer_binary//:buildozer.exe",
+              Label.RepoContext.of(
+                  RepositoryName.BAZEL_TOOLS, bazelToolsRepoMapping.getRepositoryMapping()));
+    } catch (LabelSyntaxException e) {
+      throw new IllegalStateException(e);
+    }
+    RootedPath buildozer;
+    try {
+      buildozer = RepositoryFunction.getRootedPathFromLabel(buildozerLabel, env);
+    } catch (NeedsSkyframeRestartException e) {
+      return null;
+    } catch (EvalException e) {
+      throw new IllegalStateException(e);
+    }
+
+    ImmutableSet<SkyKey> extensionsUsedByRootModule =
+        depGraphValue.getExtensionUsagesTable().columnMap().get(ModuleKey.ROOT).keySet().stream()
+            .map(SingleExtensionEvalValue::key)
+            .collect(toImmutableSet());
+    SkyframeLookupResult result = env.getValuesAndExceptions(extensionsUsedByRootModule);
+    if (env.valuesMissing()) {
+      return null;
+    }
+    for (SkyKey extension : extensionsUsedByRootModule) {
+      try {
+        result.getOrThrow(extension, ExternalDepsException.class);
+      } catch (ExternalDepsException e) {
+        if (e.getDetailedExitCode().getFailureDetail() == null
+            || !e.getDetailedExitCode()
+                .getFailureDetail()
+                .getExternalDeps()
+                .getCode()
+                .equals(FailureDetails.ExternalDeps.Code.INVALID_EXTENSION_IMPORT)) {
+          throw new BazelModTidyFunctionException(e, SkyFunctionException.Transience.PERSISTENT);
+        }
+        // This is an error bazel mod tidy can fix, so don't fail.
+      }
+    }
+
+    return BazelModTidyValue.create(
+        buildozer.asPath(),
+        MODULE_OVERRIDES.get(env),
+        IGNORE_DEV_DEPS.get(env),
+        LOCKFILE_MODE.get(env),
+        STARLARK_SEMANTICS.get(env));
+  }
+
+  static final class BazelModTidyFunctionException extends SkyFunctionException {
+
+    BazelModTidyFunctionException(ExternalDepsException cause, Transience transience) {
+      super(cause, transience);
+    }
+  }
+}