Using an ActionFileInputCache for SHA1 digests used with remote execu…

…tion.

--
MOS_MIGRATED_REVID=139613925

src/main/java/com/google/devtools/build/lib/remote/ContentDigests.java

@@ -82,9 +82,7 @@ public static ContentDigest buildDigest(byte[] digest, long size) {
   }
 
   public static String toHexString(ContentDigest digest) {
-    return digest.getSizeBytes() > 0
-        ? HashCode.fromBytes(digest.getDigest().toByteArray()).toString()
-        : "";
+    return HashCode.fromBytes(digest.getDigest().toByteArray()).toString();
   }
 
   public static String toString(ContentDigest digest) {

src/main/java/com/google/devtools/build/lib/remote/GrpcActionCache.java

@@ -629,7 +629,6 @@ public ActionResult getCachedActionResult(ActionKey actionKey) {
     ExecutionCacheReply reply = stub.getCachedResult(request);
     ExecutionCacheStatus status = reply.getStatus();
     if (!status.getSucceeded()
-        && status.getError() != ExecutionCacheStatus.ErrorCode.UNSUPPORTED
         && status.getError() != ExecutionCacheStatus.ErrorCode.MISSING_RESULT) {
       throw new RuntimeException(status.getErrorDetail());
     }
@@ -650,7 +649,8 @@ public void setCachedActionResult(ActionKey actionKey, ActionResult result)
             .build();
     ExecutionCacheSetReply reply = stub.setCachedResult(request);
     ExecutionCacheStatus status = reply.getStatus();
-    if (!status.getSucceeded()) {
+    if (!status.getSucceeded()
+        && status.getError() != ExecutionCacheStatus.ErrorCode.UNSUPPORTED) {
       throw new RuntimeException(status.getErrorDetail());
     }
   }

src/main/java/com/google/devtools/build/lib/remote/README.md

@@ -9,7 +9,8 @@ this step.
 
 - Then you run Bazel pointing to the Hazelcast server.
 
-    bazel build --hazelcast_node=localhost:5701 --spawn_strategy=remote \
+    bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
+        --hazelcast_node=localhost:5701 --spawn_strategy=remote \
         src/tools/generate_workspace:all
 
 Above command will build generate_workspace with remote spawn strategy that uses
@@ -25,6 +26,33 @@ with default configuration.
 
 - Then run Bazel pointing to the Hazelcast server and remote worker.
 
-        bazel build --hazelcast_node=localhost:5701 \
+        bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
+            --hazelcast_node=localhost:5701 \
             --remote_worker=localhost:8080 \
             --spawn_strategy=remote src/tools/generate_workspace:all
+
+# How to run a remote worker with a remote cache server.
+
+- First you need to run a standalone Hazelcast server with default
+configuration. If you already have a separate Hazelcast cluster you can skip
+this step.
+
+    java -cp third_party/hazelcast/hazelcast-3.6.4.jar \
+        com.hazelcast.core.server.StartServer
+
+- Then run the remote cache server:
+
+    bazel-bin/src/tools/remote_worker/remote_cache --listen_port 8081
+
+- The run the remote worker:
+
+    bazel-bin/src/tools/remote_worker/remote_worker \
+        --work_path=/tmp/remote --listen_port 8080
+
+- Then run Bazel pointing to the cache server and remote worker.
+
+        bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
+            --hazelcast_node=localhost:5701 \
+            --remote_worker=localhost:8080 \
+            --remote_cache=localhost:8081 \
+            --spawn_strategy=remote src/tools/generate_workspace:all

src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java

@@ -61,9 +61,8 @@ void uploadAllResults(Path execRoot, Collection<Path> files, ActionResult.Builde
       throws IOException, InterruptedException;
 
   /**
-   * Put the file contents cache if it is not already in it. No-op if the file is already stored in
-   * cache. The given path must be a full absolute path. Note: this is horribly inefficient, need to
-   * patch through an overload that uses an ActionInputFile cache to compute the digests!
+   * Put the file contents in cache if it is not already in it. No-op if the file is already stored
+   * in cache. The given path must be a full absolute path.
    *
    * @return The key for fetching the file contents blob from cache.
    */

src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java

@@ -24,6 +24,10 @@
 import com.google.devtools.build.lib.runtime.BlazeModule;
 import com.google.devtools.build.lib.runtime.Command;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystem.HashFunction;
 import com.google.devtools.common.options.OptionsBase;
 
 /** RemoteModule provides distributed cache and remote execution for Bazel. */
@@ -66,8 +70,17 @@ public void buildStarting(BuildStartingEvent event) {
       }
       // Otherwise actionCache remains null and remote caching/execution are disabled.
 
-      if (actionCache != null && RemoteWorkExecutor.isRemoteExecutionOptions(options)) {
-        workExecutor = new RemoteWorkExecutor(options);
+      if (actionCache != null) {
+        HashFunction hf = FileSystem.getDigestFunction();
+        if (hf != HashFunction.SHA1) {
+          env.getBlazeModuleEnvironment().exit(new AbruptExitException(
+              "Remote cache/execution requires SHA1 digests, got " + hf
+              + ", run with --host_jvm_args=-Dbazel.DigestFunction=SHA1",
+              ExitCode.COMMAND_LINE_ERROR));
+        }
+        if (RemoteWorkExecutor.isRemoteExecutionOptions(options)) {
+          workExecutor = new RemoteWorkExecutor(options);
+        }
       }
     } catch (InvalidConfigurationException e) {
       env.getReporter().handle(Event.warn(e.toString()));

src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java

@@ -21,8 +21,10 @@
 import com.google.devtools.build.lib.actions.ActionExecutionContext;
 import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ActionInputHelper;
+import com.google.devtools.build.lib.actions.ActionStatusMessage;
 import com.google.devtools.build.lib.actions.ExecException;
 import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
 import com.google.devtools.build.lib.actions.Spawn;
 import com.google.devtools.build.lib.actions.SpawnActionContext;
 import com.google.devtools.build.lib.actions.Spawns;
@@ -128,6 +130,11 @@ private void execLocally(
             .handle(
                 Event.warn(
                     spawn.getMnemonic() + " unsupported operation for action cache (" + e + ")"));
+      } catch (StatusRuntimeException e) {
+        actionExecutionContext
+            .getExecutor()
+            .getEventHandler()
+            .handle(Event.warn(spawn.getMnemonic() + " failed uploading results (" + e + ")"));
       }
     }
   }
@@ -147,6 +154,11 @@ private void passRemoteOutErr(ActionResult result, FileOutErr outErr) {
     }
   }
 
+  @Override
+  public String toString() {
+    return "remote";
+  }
+
   /** Executes the given {@code spawn}. */
   @Override
   public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext)
@@ -158,7 +170,10 @@ public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext)
 
     ActionKey actionKey = null;
     String mnemonic = spawn.getMnemonic();
-    EventHandler eventHandler = actionExecutionContext.getExecutor().getEventHandler();
+    Executor executor = actionExecutionContext.getExecutor();
+    EventHandler eventHandler = executor.getEventHandler();
+    executor.getEventBus().post(
+        ActionStatusMessage.runningStrategy(spawn.getResourceOwner(), "remote"));
 
     try {
       // Temporary hack: the TreeNodeRepository should be created and maintained upstream!

src/test/java/com/google/devtools/build/lib/BUILD

@@ -1032,6 +1032,7 @@ java_test(
         ":foundations_testutil",
         ":test_runner",
         ":testutil",
+        "//src/main/java/com/google/devtools/build/lib:build-base",
         "//src/main/java/com/google/devtools/build/lib:preconditions",
         "//src/main/java/com/google/devtools/build/lib:vfs",
         "//src/main/java/com/google/devtools/build/lib/actions",

src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteCache.java

@@ -188,8 +188,10 @@ public void onNext(CasUploadBlobRequest request) {
                 offset == chunk.getOffset(),
                 "Missing input chunk for digest %s",
                 ContentDigests.toString(digest));
-            chunk.getData().copyTo(blob, (int) offset);
-            offset = (offset + chunk.getData().size()) % digest.getSizeBytes();
+            if (digest.getSizeBytes() > 0) {
+              chunk.getData().copyTo(blob, (int) offset);
+              offset = (offset + chunk.getData().size()) % digest.getSizeBytes();
+            }
             if (offset == 0) {
               ContentDigest uploadedDigest = cache.uploadBlob(blob);
               Preconditions.checkArgument(