Allow the exec root to be placed outside the output base

In an attempt to achieve 'Builds without the Bytes' without losing
access to build outputs, I am experimenting with a FUSE file system that
gives access to objects stored in the CAS. In PR #11703, I added a
command line flag to let Bazel emit symbolic links pointing into this
FUSE file system, as opposed to downloading files into the exec root.

Though this change has allowed me to get quite a lot of stuff working,
there are also many build actions that break. For example, Python calls
realpath(argv[0]) to figure out its installation path. Because the FUSE
file system does not mimic the execroot, Python won't be able to find
its site packages. Similar problems hold with shared library resolution
in general.

This is why I think the only proper way we can get this to work is by
using hard links instead of symbolic links. That way the usual file
hierarchy remains intact. This, however, requires that the exec root
itself is placed on a FUSE file system. It is already possible to
achieve this by setting --output_base, but that has the downside of also
placing many other files on FUSE (e.g., the sandbox directories), which
is detrimental to performance.

This change adds a new command line flag, --exec_root_base, which can be
used to leave the output base at the regular place, but host the
exec root directory on a FUSE file system.

This change originally seemed to work all right with Bazel 3.4-3.7. In
order to make this work with Bazel master, I had to make a slight tweak
to the changes in 0c249d5. That code
added the assumption that "${output_base}/external" is always placed at
"${exec_root_base}/../external". I suspect that already causes a
regression in case a BlazeModule overrides the exec root base. While
there, rename 'execRootParent' to 'execRootBase', as it corresponds to
the exec root itself; not its parent directory.

scripts/bootstrap/compile.sh

@@ -444,6 +444,7 @@ function run_bazel_jar() {
       --batch \
       --install_base=${ARCHIVE_DIR} \
       --output_base=${OUTPUT_DIR}/out \
+      --exec_root_base=${OUTPUT_DIR}/execroot \
       --failure_detail_out=${OUTPUT_DIR}/failure_detail.rawproto \
       --output_user_root=${OUTPUT_DIR}/user_root \
       --install_md5= \

src/main/cpp/blaze.cc

@@ -455,6 +455,8 @@ static vector<string> GetServerExeArgs(const blaze_util::Path &jvm_path,
   result.push_back("--install_md5=" + install_md5);
   result.push_back("--output_base=" +
                    startup_options.output_base.AsCommandLineArgument());
+  result.push_back("--exec_root_base=" +
+                   startup_options.exec_root_base.AsCommandLineArgument());
   result.push_back("--workspace_directory=" +
                    blaze_util::ConvertPath(workspace));
   result.push_back("--default_system_javabase=" + GetSystemJavabase());
@@ -1366,6 +1368,11 @@ static void UpdateConfiguration(const string &install_md5,
     startup_options->failure_detail_out =
         startup_options->output_base.GetRelative("failure_detail.rawproto");
   }
+
+  if (startup_options->exec_root_base.IsEmpty()) {
+    startup_options->exec_root_base =
+        startup_options->output_base.GetRelative("execroot");
+  }
 }
 
 // Prepares the environment to be suitable to start a JVM.

src/main/cpp/startup_options.cc

@@ -161,6 +161,7 @@ StartupOptions::StartupOptions(const string &product_name,
   RegisterUnaryStartupFlag("max_idle_secs");
   RegisterUnaryStartupFlag("output_base");
   RegisterUnaryStartupFlag("output_user_root");
+  RegisterUnaryStartupFlag("exec_root_base");
   RegisterUnaryStartupFlag("server_jvm_out");
   RegisterUnaryStartupFlag("failure_detail_out");
 }
@@ -273,6 +274,10 @@ blaze_exit_code::ExitCode StartupOptions::ProcessArg(
                                      "--output_user_root")) != NULL) {
     output_user_root = blaze::AbsolutePathFromFlag(value);
     option_sources["output_user_root"] = rcfile;
+  } else if ((value = GetUnaryOption(arg, next_arg,
+                                     "--exec_root_base")) != NULL) {
+    exec_root_base = blaze_util::Path(blaze::AbsolutePathFromFlag(value));
+    option_sources["exec_root_base"] = rcfile;
   } else if ((value = GetUnaryOption(arg, next_arg,
                                      "--server_jvm_out")) != NULL) {
     server_jvm_out = blaze_util::Path(blaze::AbsolutePathFromFlag(value));

src/main/cpp/startup_options.h

@@ -155,6 +155,10 @@ class StartupOptions {
   // output_base.
   std::string output_user_root;
 
+  // Blaze's execroot base.  See the BlazeDirectories Java class for
+  // details.
+  blaze_util::Path exec_root_base;
+
   // Override more finegrained rc file flags and ignore them all.
   bool ignore_all_rc_files;
 

src/main/java/com/google/devtools/build/lib/BUILD

@@ -507,6 +507,7 @@ genrule(
         "$(location //src/main/java/com/google/devtools/build/lib/bazel:BazelServer) " +
         "--batch " +
         "--install_base=$${TMP} --output_base=$${TMP}/output/ --output_user_root=$${TMP} " +
+        "--exec_root_base=$${TMP}/execroot " +
         "--failure_detail_out=$${TMP}/output/failure_detail.rawproto " +
         "help everything-as-html >> $@ 2>/dev/null && " +
         "cat $(location //site:command-line-reference-suffix.html) >> $@"

src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java

@@ -36,7 +36,7 @@
 @ThreadSafe
 public class ArtifactFactory implements ArtifactResolver {
 
-  private final Path execRootParent;
+  private final Path execRootBase;
   private final Path externalSourceBase;
   private final PathFragment derivedPathPrefix;
   private boolean siblingRepositoryLayout = false;
@@ -119,14 +119,11 @@ void clear() {
   /**
    * Constructs a new artifact factory that will use a given execution root when creating artifacts.
    *
-   * @param execRootParent the execution root's parent path. This will be [output_base]/execroot.
+   * @param execRootBase the execution root's base path. This will be [output_base]/execroot.
    */
-  public ArtifactFactory(Path execRootParent, String derivedPathPrefix) {
-    this.execRootParent = execRootParent;
-    this.externalSourceBase =
-        execRootParent
-            .getParentDirectory()
-            .getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION);
+  public ArtifactFactory(Path execRootBase, Path outputBase, String derivedPathPrefix) {
+    this.execRootBase = execRootBase;
+    this.externalSourceBase = outputBase.getRelative(LabelConstants.EXTERNAL_REPOSITORY_LOCATION);
     this.derivedPathPrefix = PathFragment.create(derivedPathPrefix);
   }
 
@@ -181,18 +178,18 @@ private void validatePath(PathFragment rootRelativePath, ArtifactRoot root) {
         rootRelativePath.isAbsolute() == root.getRoot().isAbsolute(), rootRelativePath);
     Preconditions.checkArgument(!rootRelativePath.containsUplevelReferences(), rootRelativePath);
     Preconditions.checkArgument(
-        root.getRoot().asPath().startsWith(execRootParent),
-        "%s must start with %s, root = %s, root fs = %s, execRootParent fs = %s",
+        root.getRoot().asPath().startsWith(execRootBase),
+        "%s must start with %s, root = %s, root fs = %s, execRootBase fs = %s",
         root.getRoot(),
-        execRootParent,
+        execRootBase,
         root,
         root.getRoot().asPath().getFileSystem(),
-        execRootParent.getFileSystem());
+        execRootBase.getFileSystem());
     Preconditions.checkArgument(
-        !root.getRoot().asPath().equals(execRootParent),
+        !root.getRoot().asPath().equals(execRootBase),
         "%s %s %s",
         root.getRoot(),
-        execRootParent,
+        execRootBase,
         root);
     // TODO(bazel-team): this should only accept roots from derivedRoots.
     //Preconditions.checkArgument(derivedRoots.contains(root), "%s not in %s", root, derivedRoots);
@@ -202,10 +199,10 @@ private void validatePath(PathFragment rootRelativePath, ArtifactRoot root) {
    * Returns an artifact for a tool at the given root-relative path under the given root, creating
    * it if not found. This method only works for normalized, relative paths.
    *
-   * <p>The root must be below the execRootParent, and the execPath of the resulting Artifact is
+   * <p>The root must be below the execRootBase, and the execPath of the resulting Artifact is
    * computed as {@code root.getRelative(rootRelativePath).relativeTo(root.execRoot)}.
    */
-  // TODO(bazel-team): Don't allow root == execRootParent.
+  // TODO(bazel-team): Don't allow root == execRootBase.
   public Artifact.DerivedArtifact getDerivedArtifact(
       PathFragment rootRelativePath, ArtifactRoot root, ArtifactOwner owner) {
     return getDerivedArtifact(rootRelativePath, root, owner, /*contentBasedPath=*/ false);
@@ -232,7 +229,7 @@ public Artifact.DerivedArtifact getDerivedArtifact(
    * root-relative path under the given root, creating it if not found. This method only works for
    * normalized, relative paths.
    *
-   * <p>The root must be below the execRootParent, and the execPath of the resulting Artifact is
+   * <p>The root must be below the execRootBase, and the execPath of the resulting Artifact is
    * computed as {@code root.getRelative(rootRelativePath).relativeTo(root.execRoot)}.
    */
   public Artifact.DerivedArtifact getFilesetArtifact(
@@ -251,7 +248,7 @@ public Artifact.DerivedArtifact getFilesetArtifact(
    * Returns an artifact that represents a TreeArtifact; that is, a directory containing some tree
    * of ArtifactFiles unknown at analysis time.
    *
-   * <p>The root must be below the execRootParent, and the execPath of the resulting Artifact is
+   * <p>The root must be below the execRootBase, and the execPath of the resulting Artifact is
    * computed as {@code root.getRelative(rootRelativePath).relativeTo(root.execRoot)}.
    */
   public Artifact.SpecialArtifact getTreeArtifact(
@@ -270,7 +267,7 @@ public Artifact.SpecialArtifact getTreeArtifact(
    * Returns an artifact that represents an unresolved symlink; that is, an artifact whose value is
    * a symlink and is never dereferenced.
    *
-   * <p>The root must be below the execRootParent, and the execPath of the resulting Artifact is
+   * <p>The root must be below the execRootBase, and the execPath of the resulting Artifact is
    * computed as {@code root.getRelative(rootRelativePath).relativeTo(root.execRoot)}.
    */
   public Artifact.SpecialArtifact getSymlinkArtifact(

src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java

@@ -1143,6 +1143,7 @@ private static BlazeRuntime newRuntime(
     PathFragment outputUserRoot = startupOptions.outputUserRoot;
     PathFragment installBase = startupOptions.installBase;
     PathFragment outputBase = startupOptions.outputBase;
+    PathFragment execRootBase = startupOptions.execRootBase;
 
     maybeForceJNIByGettingPid(installBase); // Must be before first use of JNI.
 
@@ -1161,12 +1162,16 @@ private static BlazeRuntime newRuntime(
       throw new IllegalArgumentException(
           "Bad --output_base option specified: '" + outputBase + "'");
     }
+    if (execRootBase != null && !execRootBase.isAbsolute()) { // (includes "" default case)
+      throw new IllegalArgumentException(
+          "Bad --exec_root_base option specified: '" + execRootBase + "'");
+    }
 
     FileSystem fs = null;
     Path execRootBasePath = null;
     for (BlazeModule module : blazeModules) {
       BlazeModule.ModuleFileSystem moduleFs =
-          module.getFileSystem(options, outputBase.getRelative(ServerDirectories.EXECROOT));
+          module.getFileSystem(options, execRootBase);
       if (moduleFs != null) {
         execRootBasePath = moduleFs.virtualExecRootBase();
         Preconditions.checkState(fs == null, "more than one module returns a file system");
@@ -1226,7 +1231,7 @@ private static BlazeRuntime newRuntime(
     Path installBasePath = fs.getPath(installBase);
     Path outputBasePath = fs.getPath(outputBase);
     if (execRootBasePath == null) {
-      execRootBasePath = outputBasePath.getRelative(ServerDirectories.EXECROOT);
+      execRootBasePath = fs.getPath(execRootBase);
     }
     Path workspaceDirectoryPath = null;
     if (!workspaceDirectory.equals(PathFragment.EMPTY_FRAGMENT)) {

src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java

@@ -142,6 +142,20 @@ public String getTypeDescription() {
               + "shared between collaborating users.")
   public PathFragment outputUserRoot;
 
+  @Option(
+      name = "exec_root_base",
+      defaultValue = "null", // NOTE: only for documentation, value is always passed by the client.
+      documentationCategory = OptionDocumentationCategory.BAZEL_CLIENT_OPTIONS,
+      effectTags = {OptionEffectTag.AFFECTS_OUTPUTS, OptionEffectTag.LOSES_INCREMENTAL_STATE},
+      converter = OptionsUtils.PathFragmentConverter.class,
+      valueHelp = "<path>",
+      help =
+          "If set, specifies the location at which the exec root directory is placed. Otherwise, "
+              + "the location will be ${OUTPUT_BASE}/execroot. This option may for example be used "
+              + "to place the exec root directory on a FUSE file system, while leaving other files "
+              + "stored in the output base (e.g., sandbox directories) on a local file system.")
+  public PathFragment execRootBase;
+
   /**
    * Note: This option is only used by the C++ client, never by the Java server. It is included here
    * to make sure that the option is documented in the help output, which is auto-generated by Java

src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java

@@ -164,7 +164,8 @@ public SkyframeBuildView(
     this.factory = new ConfiguredTargetFactory(ruleClassProvider);
     this.artifactFactory =
         new ArtifactFactory(
-            /* execRootParent= */ directories.getExecRootBase(),
+            /* execRootBase= */ directories.getExecRootBase(),
+            /* outputBase= */ directories.getOutputBase(),
             directories.getRelativeOutputPath());
     this.skyframeExecutor = skyframeExecutor;
     this.ruleClassProvider = ruleClassProvider;

src/test/java/com/google/devtools/build/lib/actions/ArtifactFactoryTest.java

@@ -71,7 +71,8 @@ public class ArtifactFactoryTest {
 
   @Before
   public final void createFiles() throws Exception  {
-    execRoot = scratch.dir("/output/workspace");
+    Path outputBase = scratch.dir("/output/workspace");
+    execRoot = scratch.dir("/output/workspace/execroot");
     clientRoot = Root.fromPath(scratch.dir("/client/workspace"));
     clientRoRoot = Root.fromPath(scratch.dir("/client/RO/workspace"));
     alienRoot = Root.fromPath(scratch.dir("/client/workspace"));
@@ -89,7 +90,7 @@ public final void createFiles() throws Exception  {
     alienPackage = PackageIdentifier.create("@alien", alienPath);
     alienRelative = alienPath.getRelative("alien.txt");
 
-    artifactFactory = new ArtifactFactory(execRoot.getParentDirectory(), "bazel-out");
+    artifactFactory = new ArtifactFactory(execRoot, outputBase, "bazel-out");
     setupRoots();
   }
 

src/test/java/com/google/devtools/build/lib/actions/ArtifactTest.java

@@ -58,13 +58,15 @@
 @RunWith(JUnit4.class)
 public class ArtifactTest {
   private Scratch scratch;
+  private Path outputBase;
   private Path execDir;
   private ArtifactRoot rootDir;
   private final ActionKeyContext actionKeyContext = new ActionKeyContext();
 
   @Before
   public final void setRootDir() throws Exception  {
     scratch = new Scratch();
+    outputBase = scratch.dir("/base");
     execDir = scratch.dir("/base/exec");
     rootDir = ArtifactRoot.asDerivedRoot(execDir, "root");
   }
@@ -299,7 +301,7 @@ public void testCodecRecyclesSourceArtifactInstances() throws Exception {
     Root root = Root.fromPath(scratch.dir("/"));
     ArtifactRoot artifactRoot = ArtifactRoot.asSourceRoot(root);
     ArtifactFactory artifactFactory =
-        new ArtifactFactory(execDir.getParentDirectory(), "blaze-out");
+        new ArtifactFactory(execDir, outputBase, "blaze-out");
     ArtifactResolverSupplier artifactResolverSupplierForTest =
         new ArtifactResolverSupplier() {
           @Override

src/test/shell/bazel/execroot_test.sh

@@ -208,4 +208,24 @@ EOF
     || fail "build failed"
 }
 
+function test_exec_root_base {
+  mkdir src
+  cat > src/WORKSPACE << 'EOF'
+workspace(name = "myworkspace")
+EOF
+  cat > src/BUILD << 'EOF'
+genrule(
+    name = "foo",
+    outs = ["foo.out"],
+    cmd = "touch $@",
+)
+EOF
+
+  execroot="$(pwd)/execroot"
+  (cd src && bazel --exec_root_base="${execroot}" build //:foo) ||
+      fail "Build failed"
+  test -f ${execroot}/myworkspace/bazel-out/*/bin/foo.out ||
+      fail "Bazel did not store the exec root in the right place"
+}
+
 run_suite "execution root tests"