Find runfiles in directories that are themselves runfiles

When a target has a runfile that is contained in a directory that is itself one
of its runfiles, the runfile will be shadowed by the SymlinkEntry for the
directory. While this still allows to resolve the file in the runfiles symlink
tree, a manifest-based lookup will fail. Since the manifest is used to create the
symlink tree for which it is important that no path is a prefix of another, this
can only be fixed in the runfiles libraries.

This commit extends the lookup logic of the Bash, C++, Java, and Python runfiles
libraries to also find runfiles contained within directories that are themselves
runfiles. It does so by searching the manifest not only for the exact provided
rlocation path, but also for all path prefixes. If a prefix is looked up
successfully, the corresponding suffix is resolved relative to the looked up
path.

tools/bash/runfiles/runfiles.bash

@@ -122,16 +122,51 @@ function rlocation() {
         echo >&2 "INFO[runfiles.bash]: rlocation($1): looking in RUNFILES_MANIFEST_FILE ($RUNFILES_MANIFEST_FILE)"
       fi
       local -r result=$(grep -m1 "^$1 " "${RUNFILES_MANIFEST_FILE}" | cut -d ' ' -f 2-)
-      if [[ -e "${result:-/dev/null}" ]]; then
-        if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then
-          echo >&2 "INFO[runfiles.bash]: rlocation($1): found in manifest as ($result)"
-        fi
-        echo "$result"
-      else
+      if [[ -z "$result" ]]; then
+        # If path references a runfile that lies under a directory that itself
+        # is a runfile, then only the directory is listed in the manifest. Look
+        # up all prefixes of path in the manifest and append the relative path
+        # from the prefix if there is a match.
+        local prefix="$1"
+        local prefix_result=
+        local new_prefix=
+        while true; do
+          new_prefix="${prefix%/*}"
+          [[ "$new_prefix" == "$prefix" ]] && break
+          prefix="$new_prefix"
+          prefix_result=$(grep -m1 "^$prefix " "${RUNFILES_MANIFEST_FILE}" | cut -d ' ' -f 2-)
+          [[ -z "$prefix_result" ]] && continue
+          local -r candidate="${prefix_result}${1#"${prefix}"}"
+          if [[ -e "$candidate" ]]; then
+            if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then
+              echo >&2 "INFO[runfiles.bash]: rlocation($1): found in manifest as ($candidate) via prefix ($prefix)"
+            fi
+            echo "$candidate"
+            return 0
+          fi
+          # At this point, the manifest lookup of prefix has been successful,
+          # but the file at the relative path given by the suffix does not
+          # exist. We do not continue the lookup with a shorter prefix for two
+          # reasons:
+          # 1. Manifests generated by Bazel never contain a path that is a
+          #    prefix of another path.
+          # 2. Runfiles libraries for other languages do not check for file
+          #    existence and would have returned the non-existent path. It seems
+          #    better to return no path rather than a potentially different,
+          #    non-empty path.
+          break
+        done
         if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then
           echo >&2 "INFO[runfiles.bash]: rlocation($1): not found in manifest"
         fi
         echo ""
+      else
+        if [[ -e "$result" ]]; then
+          if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then
+            echo >&2 "INFO[runfiles.bash]: rlocation($1): found in manifest as ($result)"
+          fi
+          echo "$result"
+        fi
       fi
     else
       if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then

tools/bash/runfiles/runfiles_test.bash

@@ -139,10 +139,14 @@ function test_init_manifest_based_runfiles() {
 a/b $tmpdir/c/d
 e/f $tmpdir/g h
 y $tmpdir/y
+c/dir $tmpdir/dir
 EOF
   mkdir "${tmpdir}/c"
   mkdir "${tmpdir}/y"
+  mkdir -p "${tmpdir}/dir/deeply/nested"
   touch "${tmpdir}/c/d" "${tmpdir}/g h"
+  touch "${tmpdir}/dir/file"
+  touch "${tmpdir}/dir/deeply/nested/file"
 
   export RUNFILES_DIR=
   export RUNFILES_MANIFEST_FILE=$tmpdir/foo.runfiles_manifest
@@ -153,10 +157,18 @@ EOF
   [[ "$(rlocation a/b)" == "$tmpdir/c/d" ]] || fail
   [[ "$(rlocation e/f)" == "$tmpdir/g h" ]] || fail
   [[ "$(rlocation y)" == "$tmpdir/y" ]] || fail
-  rm -r "$tmpdir/c/d" "$tmpdir/g h" "$tmpdir/y"
+  [[ "$(rlocation c)" == "" ]] || fail
+  [[ "$(rlocation c/di)" == "" ]] || fail
+  [[ "$(rlocation c/dir)" == "$tmpdir/dir" ]] || fail
+  [[ "$(rlocation c/dir/file)" == "$tmpdir/dir/file" ]] || fail
+  [[ "$(rlocation c/dir/deeply/nested/file)" == "$tmpdir/dir/deeply/nested/file" ]] || fail
+  rm -r "$tmpdir/c/d" "$tmpdir/g h" "$tmpdir/y" "$tmpdir/dir"
   [[ -z "$(rlocation a/b)" ]] || fail
   [[ -z "$(rlocation e/f)" ]] || fail
   [[ -z "$(rlocation y)" ]] || fail
+  [[ -z "$(rlocation c/dir)" ]] || fail
+  [[ -z "$(rlocation c/dir/file)" ]] || fail
+  [[ -z "$(rlocation c/dir/deeply/nested/file)" ]] || fail
 }
 
 function test_manifest_based_envvars() {

tools/cpp/runfiles/runfiles_src.cc

@@ -25,6 +25,7 @@
 #include <unistd.h>
 #endif  // _WIN32
 
+#include <algorithm>
 #include <fstream>
 #include <functional>
 #include <map>
@@ -176,9 +177,24 @@ string Runfiles::Rlocation(const string& path) const {
   if (IsAbsolute(path)) {
     return path;
   }
-  const auto value = runfiles_map_.find(path);
-  if (value != runfiles_map_.end()) {
-    return value->second;
+  const auto exact_match = runfiles_map_.find(path);
+  if (exact_match != runfiles_map_.end()) {
+    return exact_match->second;
+  }
+  if (!runfiles_map_.empty()) {
+    // If path references a runfile that lies under a directory that itself is a
+    // runfile, then only the directory is listed in the manifest. Look up all
+    // prefixes of path in the manifest and append the relative path from the
+    // prefix to the looked up path.
+    std::size_t prefix_end = path.size();
+    while ((prefix_end = path.find_last_of('/', prefix_end - 1)) !=
+           string::npos) {
+      const string prefix = path.substr(0, prefix_end);
+      const auto prefix_match = runfiles_map_.find(prefix);
+      if (prefix_match != runfiles_map_.end()) {
+        return prefix_match->second + "/" + path.substr(prefix_end + 1);
+      }
+    }
   }
   if (!directory_.empty()) {
     return directory_ + "/" + path;

tools/cpp/runfiles/runfiles_test.cc

@@ -252,6 +252,8 @@ TEST_F(RunfilesTest, ManifestBasedRunfilesRlocationAndEnvVars) {
   EXPECT_EQ(r->Rlocation("/Foo"), "/Foo");
   EXPECT_EQ(r->Rlocation("c:/Foo"), "c:/Foo");
   EXPECT_EQ(r->Rlocation("c:\\Foo"), "c:\\Foo");
+  EXPECT_EQ(r->Rlocation("a/b/file"), "c/d/file");
+  EXPECT_EQ(r->Rlocation("a/b/deeply/nested/file"), "c/d/deeply/nested/file");
 }
 
 TEST_F(RunfilesTest, DirectoryBasedRunfilesRlocationAndEnvVars) {
@@ -316,6 +318,8 @@ TEST_F(RunfilesTest, ManifestAndDirectoryBasedRunfilesRlocationAndEnvVars) {
   EXPECT_EQ(r->Rlocation("/Foo"), "/Foo");
   EXPECT_EQ(r->Rlocation("c:/Foo"), "c:/Foo");
   EXPECT_EQ(r->Rlocation("c:\\Foo"), "c:\\Foo");
+  EXPECT_EQ(r->Rlocation("a/b/file"), "c/d/file");
+  EXPECT_EQ(r->Rlocation("a/b/deeply/nested/file"), "c/d/deeply/nested/file");
   AssertEnvvars(*r, mf->Path(), dir);
 }
 

tools/java/runfiles/Runfiles.java

@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -227,7 +228,21 @@ private static String findRunfilesDir(String manifest) {
 
     @Override
     public String rlocationChecked(String path) {
-      return runfiles.get(path);
+      String exactMatch = runfiles.get(path);
+      if (exactMatch != null) {
+        return exactMatch;
+      }
+      // If path references a runfile that lies under a directory that itself is a runfile, then
+      // only the directory is listed in the manifest. Look up all prefixes of path in the manifest
+      // and append the relative path from the prefix if there is a match.
+      int prefixEnd = path.length();
+      while ((prefixEnd = path.lastIndexOf('/', prefixEnd - 1)) != -1) {
+        String prefixMatch = runfiles.get(path.substring(0, prefixEnd));
+        if (prefixMatch != null) {
+          return prefixMatch + '/' + path.substring(prefixEnd + 1);
+        }
+      }
+      return null;
     }
 
     @Override

tools/java/runfiles/testing/RunfilesTest.java

@@ -25,6 +25,7 @@
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.Map;
 import javax.annotation.Nullable;
@@ -249,10 +250,15 @@ public void testManifestBasedRlocation() throws Exception {
         new MockFile(
             ImmutableList.of(
                 "Foo/runfile1 C:/Actual Path\\runfile1",
-                "Foo/Bar/runfile2 D:\\the path\\run file 2.txt"))) {
+                "Foo/Bar/runfile2 D:\\the path\\run file 2.txt",
+                "Foo/Bar/Dir E:\\Actual Path\\Directory"))) {
       Runfiles r = Runfiles.createManifestBasedForTesting(mf.path.toString());
       assertThat(r.rlocation("Foo/runfile1")).isEqualTo("C:/Actual Path\\runfile1");
       assertThat(r.rlocation("Foo/Bar/runfile2")).isEqualTo("D:\\the path\\run file 2.txt");
+      assertThat(r.rlocation("Foo/Bar/Dir")).isEqualTo("E:\\Actual Path\\Directory");
+      assertThat(r.rlocation("Foo/Bar/Dir/File")).isEqualTo("E:\\Actual Path\\Directory/File");
+      assertThat(r.rlocation("Foo/Bar/Dir/Deeply/Nested/File")).isEqualTo(
+          "E:\\Actual Path\\Directory/Deeply/Nested/File");
       assertThat(r.rlocation("unknown")).isNull();
     }
   }

tools/python/runfiles/runfiles.py

@@ -174,7 +174,21 @@ def __init__(self, path):
     self._runfiles = _ManifestBased._LoadRunfiles(path)
 
   def RlocationChecked(self, path):
-    return self._runfiles.get(path)
+    exact_match = self._runfiles.get(path)
+    if exact_match:
+      return exact_match
+    # If path references a runfile that lies under a directory that itself is a
+    # runfile, then only the directory is listed in the manifest. Look up all
+    # prefixes of path in the manifest and append the relative path from the
+    # prefix to the looked up path.
+    prefix_end = len(path)
+    while True:
+      prefix_end = path.rfind("/", 0, prefix_end - 1)
+      if prefix_end == -1:
+        return None
+      prefix_match = self._runfiles.get(path[0:prefix_end])
+      if prefix_match:
+        return prefix_match + '/' + path[prefix_end + 1:]
 
   @staticmethod
   def _LoadRunfiles(path):

tools/python/runfiles/runfiles_test.py

@@ -140,14 +140,19 @@ def testFailsToCreateAnyRunfilesBecauseEnvvarsAreNotDefined(self):
 
   def testManifestBasedRlocation(self):
     with _MockFile(contents=[
-        "Foo/runfile1", "Foo/runfile2 C:/Actual Path\\runfile2",
-        "Foo/Bar/runfile3 D:\\the path\\run file 3.txt"
+      "Foo/runfile1", "Foo/runfile2 C:/Actual Path\\runfile2",
+      "Foo/Bar/runfile3 D:\\the path\\run file 3.txt",
+      "Foo/Bar/Dir E:\\Actual Path\\Directory",
     ]) as mf:
       r = runfiles.CreateManifestBased(mf.Path())
       self.assertEqual(r.Rlocation("Foo/runfile1"), "Foo/runfile1")
       self.assertEqual(r.Rlocation("Foo/runfile2"), "C:/Actual Path\\runfile2")
       self.assertEqual(
           r.Rlocation("Foo/Bar/runfile3"), "D:\\the path\\run file 3.txt")
+      self.assertEqual(r.Rlocation("Foo/Bar/Dir/runfile4"),
+                       "E:\\Actual Path\\Directory/runfile4")
+      self.assertEqual(r.Rlocation("Foo/Bar/Dir/Deeply/Nested/runfile4"),
+                       "E:\\Actual Path\\Directory/Deeply/Nested/runfile4")
       self.assertIsNone(r.Rlocation("unknown"))
       if RunfilesTest.IsWindows():
         self.assertEqual(r.Rlocation("c:/foo"), "c:/foo")