Open source DynamicExecutionModule

RELNOTES: Dynamic execution is now available with --experimental_spawn_strategy. Dynamic execution allows a build action to run locally and remotely simultaneously, and Bazel picks the fastest action. This provides the best of both worlds: faster clean builds than pure local builds, and faster incremental builds than pure remote builds.
PiperOrigin-RevId: 222446721

src/main/java/com/google/devtools/build/lib/BUILD

@@ -30,6 +30,7 @@ filegroup(
         "//src/main/java/com/google/devtools/build/lib/collect/nestedset:srcs",
         "//src/main/java/com/google/devtools/build/lib/collect:srcs",
         "//src/main/java/com/google/devtools/build/lib/concurrent:srcs",
+        "//src/main/java/com/google/devtools/build/lib/dynamic:srcs",
         "//src/main/java/com/google/devtools/build/lib/exec/apple:srcs",
         "//src/main/java/com/google/devtools/build/lib/exec/local:srcs",
         "//src/main/java/com/google/devtools/build/lib/graph:srcs",
@@ -766,6 +767,7 @@ java_library(
         ":build-base",
         "//src/main/java/com/google/devtools/build/lib/bazel/debug:workspace-rule-module",
         "//src/main/java/com/google/devtools/build/lib/buildeventservice",
+        "//src/main/java/com/google/devtools/build/lib/dynamic",
         "//src/main/java/com/google/devtools/build/lib/metrics:metrics_module",
         "//src/main/java/com/google/devtools/build/lib/profiler/callcounts:callcounts_module",
         "//src/main/java/com/google/devtools/build/lib/profiler/memory:allocationtracker_module",

src/main/java/com/google/devtools/build/lib/bazel/Bazel.java

@@ -55,6 +55,7 @@ public final class Bazel {
           com.google.devtools.build.lib.standalone.StandaloneModule.class,
           com.google.devtools.build.lib.sandbox.SandboxModule.class,
           com.google.devtools.build.lib.runtime.BuildSummaryStatsModule.class,
+          com.google.devtools.build.lib.dynamic.DynamicExecutionModule.class,
           com.google.devtools.build.lib.bazel.rules.BazelRulesModule.class,
           com.google.devtools.build.lib.bazel.rules.BazelStrategyModule.class,
           com.google.devtools.build.lib.buildeventservice.BazelBuildEventServiceModule.class,

src/main/java/com/google/devtools/build/lib/dynamic/BUILD

@@ -0,0 +1,25 @@
+package(default_visibility = ["//src:__subpackages__"])
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["**"]),
+    visibility = ["//src/main/java/com/google/devtools/build/lib:__pkg__"],
+)
+
+java_library(
+    name = "dynamic",
+    srcs = glob(["*.java"]),
+    deps = [
+        "//src/main/java/com/google/devtools/build/lib:build-base",
+        "//src/main/java/com/google/devtools/build/lib:events",
+        "//src/main/java/com/google/devtools/build/lib:io",
+        "//src/main/java/com/google/devtools/build/lib:runtime",
+        "//src/main/java/com/google/devtools/build/lib/actions",
+        "//src/main/java/com/google/devtools/build/lib/concurrent",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/common/options",
+        "//third_party:auto_value",
+        "//third_party:guava",
+        "//third_party:jsr305",
+    ],
+)

src/main/java/com/google/devtools/build/lib/dynamic/DynamicExecutionModule.java

@@ -0,0 +1,112 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.dynamic;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.ExecutorInitException;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.actions.Spawns;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.concurrent.ExecutorUtil;
+import com.google.devtools.build.lib.exec.ExecutionPolicy;
+import com.google.devtools.build.lib.exec.ExecutorBuilder;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.common.options.OptionsBase;
+import java.util.Arrays;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * {@link BlazeModule} providing support for dynamic spawn execution and scheduling.
+ */
+public class DynamicExecutionModule extends BlazeModule {
+  private ExecutorService executorService;
+
+  @Override
+  public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
+    return "build".equals(command.name())
+        ? ImmutableList.<Class<? extends OptionsBase>>of(DynamicExecutionOptions.class)
+        : ImmutableList.<Class<? extends OptionsBase>>of();
+  }
+
+  @Override
+  public void beforeCommand(CommandEnvironment env) {
+    executorService =
+        Executors.newCachedThreadPool(
+            new ThreadFactoryBuilder().setNameFormat("dynamic-execution-thread-%d").build());
+    env.getEventBus().register(this);
+  }
+
+  /**
+   * Adds a strategy that backs the dynamic scheduler to the executor builder.
+   *
+   * @param builder the executor builder to modify
+   * @param name the name of the strategy
+   * @param flagName name of the flag the strategy came from; used for error reporting
+   *     purposes only
+   * @throws ExecutorInitException if the provided strategy would cause a scheduling cycle
+   */
+  private static void addBackingStrategy(ExecutorBuilder builder, String name, String flagName)
+      throws ExecutorInitException {
+    ExecutionStrategy strategy = DynamicSpawnStrategy.class.getAnnotation(ExecutionStrategy.class);
+    checkNotNull(strategy, "DynamicSpawnStrategy lacks expected ExecutionStrategy annotation");
+
+    if (Arrays.asList(strategy.name()).contains(name)) {
+      throw new ExecutorInitException("Cannot use strategy " + name + " in flag " + flagName
+          + " as it would create a cycle during execution");
+    }
+
+    builder.addStrategyByContext(SpawnActionContext.class, name);
+  }
+
+  @Override
+  public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorBuilder builder)
+      throws ExecutorInitException {
+    DynamicExecutionOptions options = env.getOptions().getOptions(DynamicExecutionOptions.class);
+    if (options.internalSpawnScheduler) {
+      builder.addActionContext(
+          new DynamicSpawnStrategy(executorService, options, this::getExecutionPolicy));
+      builder.addStrategyByContext(SpawnActionContext.class, "dynamic");
+      addBackingStrategy(builder, options.dynamicLocalStrategy, "--dynamic_local_strategy");
+      addBackingStrategy(builder, options.dynamicRemoteStrategy, "--dynamic_remote_strategy");
+      addBackingStrategy(builder, options.dynamicWorkerStrategy, "--dynamic_worker_strategy");
+    }
+  }
+
+  /**
+   * Use the {@link Spawn} metadata to determine if it can be executed locally, remotely, or both.
+   * @param spawn the {@link Spawn} action
+   * @return the {@link ExecutionPolicy} containing local/remote execution policies
+   */
+  protected ExecutionPolicy getExecutionPolicy(Spawn spawn) {
+    if (!Spawns.mayBeExecutedRemotely(spawn)) {
+      return ExecutionPolicy.LOCAL_EXECUTION_ONLY;
+    }
+
+    return ExecutionPolicy.ANYWHERE;
+  }
+
+  @Override
+  public void afterCommand() {
+    ExecutorUtil.interruptibleShutdown(executorService);
+    executorService = null;
+  }
+}

src/main/java/com/google/devtools/build/lib/dynamic/DynamicExecutionOptions.java

@@ -0,0 +1,94 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.dynamic;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionDocumentationCategory;
+import com.google.devtools.common.options.OptionEffectTag;
+import com.google.devtools.common.options.OptionsBase;
+
+/**
+ * Options related to dynamic spawn execution.
+ */
+public class DynamicExecutionOptions extends OptionsBase {
+
+  @Option(
+      name = "experimental_spawn_scheduler",
+      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      defaultValue = "null",
+      help =
+          "Run actions locally instead of remotely for incremental builds as long as enough "
+              + "resources are available to execute all runnable actions in parallel.",
+      expansion = {"--internal_spawn_scheduler", "--spawn_strategy=dynamic"})
+  public Void experimentalSpawnScheduler;
+
+  @Option(
+    name = "internal_spawn_scheduler",
+    documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+    effectTags = {OptionEffectTag.UNKNOWN},
+    defaultValue = "false",
+    help =
+        "Placeholder option so that we can tell in Blaze whether the spawn scheduler was "
+            + "enabled."
+  )
+  public boolean internalSpawnScheduler;
+
+  @Option(
+    name = "dynamic_local_strategy",
+    documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+    effectTags = {OptionEffectTag.UNKNOWN},
+    defaultValue = "sandboxed",
+    help = "Strategy to use when the dynamic spawn scheduler decides to run an action locally."
+  )
+  public String dynamicLocalStrategy;
+
+  @Option(
+      name = "dynamic_remote_strategy",
+      documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      defaultValue = "remote",
+      help = "Strategy to use when the dynamic spawn scheduler decides to run an action remotely."
+  )
+  public String dynamicRemoteStrategy;
+
+  @Option(
+      name = "dynamic_worker_strategy",
+      documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      defaultValue = "worker",
+      help = "Strategy to use when the dynamic spawn scheduler decides to run an action in a"
+          + " worker."
+  )
+  public String dynamicWorkerStrategy;
+
+  @Option(
+    name = "experimental_local_execution_delay",
+    documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+    effectTags = {OptionEffectTag.UNKNOWN},
+    defaultValue = "1000",
+    help =
+        "How many milliseconds should local execution be delayed, if remote execution was faster"
+            + " during a build at least once?"
+  )
+  public int localExecutionDelay;
+
+  @Option(
+    name = "experimental_debug_spawn_scheduler",
+    documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+    effectTags = {OptionEffectTag.UNKNOWN},
+    defaultValue = "false"
+  )
+  public boolean debugSpawnScheduler;
+}