From 7fed3de0750660970c5ce7717c761803822d7f07 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Fri, 11 Nov 2022 06:55:05 -0800 Subject: [PATCH] Make Bash runfiles library repo mapping aware Work towards #16124 Closes #16693. PiperOrigin-RevId: 487811689 Change-Id: Iec5ef01ae09394372b87e5cb697ceb728563d2dc --- src/test/py/bazel/bzlmod/bazel_module_test.py | 60 ++++ src/test/shell/bazel/BUILD | 1 + src/test/shell/bazel/bazel_rules_test.sh | 292 ++++++++++++++++++ tools/bash/runfiles/runfiles.bash | 268 ++++++++++++---- tools/bash/runfiles/runfiles_test.bash | 176 +++++++++++ 5 files changed, 738 insertions(+), 59 deletions(-) diff --git a/src/test/py/bazel/bzlmod/bazel_module_test.py b/src/test/py/bazel/bzlmod/bazel_module_test.py index 5da5f7802962f1..b2e20d4c82e71d 100644 --- a/src/test/py/bazel/bzlmod/bazel_module_test.py +++ b/src/test/py/bazel/bzlmod/bazel_module_test.py @@ -814,5 +814,65 @@ def testCppRunfilesLibraryRepoMapping(self): allow_failure=True) self.AssertExitCode(exit_code, 0, stderr, stdout) + def testBashRunfilesLibraryRepoMapping(self): + self.main_registry.setModuleBasePath('projects') + projects_dir = self.main_registry.projects + + self.main_registry.createLocalPathModule('data', '1.0', 'data') + projects_dir.joinpath('data').mkdir(exist_ok=True) + scratchFile(projects_dir.joinpath('data', 'WORKSPACE')) + scratchFile(projects_dir.joinpath('data', 'foo.txt'), ['hello']) + scratchFile( + projects_dir.joinpath('data', 'BUILD'), ['exports_files(["foo.txt"])']) + + self.main_registry.createLocalPathModule('test', '1.0', 'test', + {'data': '1.0'}) + projects_dir.joinpath('test').mkdir(exist_ok=True) + scratchFile(projects_dir.joinpath('test', 'WORKSPACE')) + scratchFile( + projects_dir.joinpath('test', 'BUILD'), [ + 'sh_test(', + ' name = "test",', + ' srcs = ["test.sh"],', + ' data = ["@data//:foo.txt"],', + ' args = ["$(rlocationpath @data//:foo.txt)"],', + ' deps = ["@bazel_tools//tools/bash/runfiles"],', + ')', + ]) + test_script = projects_dir.joinpath('test', 'test.sh') + scratchFile( + test_script, """#!/usr/bin/env bash +# --- begin runfiles.bash initialization v2 --- +# Copy-pasted from the Bazel Bash runfiles library v2. +set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v2 --- +[[ -f "$(rlocation $1)" ]] || exit 1 +[[ -f "$(rlocation data/foo.txt)" ]] || exit 2 +""".splitlines()) + os.chmod(test_script, 0o755) + + self.ScratchFile('MODULE.bazel', ['bazel_dep(name="test",version="1.0")']) + self.ScratchFile('WORKSPACE') + + # Run sandboxed on Linux and macOS. + exit_code, stderr, stdout = self.RunBazel([ + 'test', '@test//:test', '--test_output=errors', + '--test_env=RUNFILES_LIB_DEBUG=1' + ], + allow_failure=True) + self.AssertExitCode(exit_code, 0, stderr, stdout) + # Run unsandboxed on all platforms. + exit_code, stderr, stdout = self.RunBazel( + ['run', '@test//:test'], + allow_failure=True, + env_add={'RUNFILES_LIB_DEBUG': '1'}) + self.AssertExitCode(exit_code, 0, stderr, stdout) + if __name__ == '__main__': unittest.main() diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 7fb9e679008127..ee97c61db6d2d8 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -357,6 +357,7 @@ sh_test( "@bazel_tools//tools/bash/runfiles", ], shard_count = 3, + tags = ["requires-network"], ) sh_test( diff --git a/src/test/shell/bazel/bazel_rules_test.sh b/src/test/shell/bazel/bazel_rules_test.sh index f4d69224e261e4..cc49a1fe4728dc 100755 --- a/src/test/shell/bazel/bazel_rules_test.sh +++ b/src/test/shell/bazel/bazel_rules_test.sh @@ -743,4 +743,296 @@ EOF bazel run //pkg:my_executable >$TEST_log 2>&1 || fail "Binary should have exit code 0" } +function setup_bash_runfiles_current_repository() { + touch MODULE.bazel + + cat >> WORKSPACE <<'EOF' +local_repository( + name = "other_repo", + path = "other_repo", +) +EOF + + mkdir -p pkg + cat > pkg/BUILD.bazel <<'EOF' +sh_library( + name = "library", + srcs = ["library.sh"], + deps = ["@bazel_tools//tools/bash/runfiles"], + visibility = ["//visibility:public"], +) +sh_binary( + name = "binary", + srcs = ["binary.sh"], + deps = [ + ":library", + "@other_repo//pkg:library2", + "@bazel_tools//tools/bash/runfiles", + ], +) +sh_test( + name = "test", + srcs = ["test.sh"], + deps = [ + ":library", + "@other_repo//pkg:library2", + "@bazel_tools//tools/bash/runfiles", + ], +) +EOF + + cat > pkg/library.sh <<'EOF' +#!/usr/bin/env bash +# --- begin runfiles.bash initialization v2 --- +# Copy-pasted from the Bazel Bash runfiles library v2. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v2 --- + +function library() { + echo "in pkg/library.sh: '$(runfiles_current_repository)'" +} +export -f library +EOF + chmod +x pkg/library.sh + + cat > pkg/binary.sh <<'EOF' +#!/usr/bin/env bash +# --- begin runfiles.bash initialization v2 --- +# Copy-pasted from the Bazel Bash runfiles library v2. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v2 --- + +echo "in pkg/binary.sh: '$(runfiles_current_repository)'" +source $(rlocation _main/pkg/library.sh) +library +source $(rlocation other_repo/pkg/library2.sh) +library2 +EOF + chmod +x pkg/binary.sh + + cat > pkg/test.sh <<'EOF' +#!/usr/bin/env bash +# --- begin runfiles.bash initialization v2 --- +# Copy-pasted from the Bazel Bash runfiles library v2. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v2 --- + +echo "in pkg/test.sh: '$(runfiles_current_repository)'" +source $(rlocation _main/pkg/library.sh) +library +source $(rlocation other_repo/pkg/library2.sh) +library2 +EOF + chmod +x pkg/test.sh + + mkdir -p other_repo + touch other_repo/WORKSPACE + + mkdir -p other_repo/pkg + cat > other_repo/pkg/BUILD.bazel <<'EOF' +sh_library( + name = "library2", + srcs = ["library2.sh"], + deps = ["@bazel_tools//tools/bash/runfiles"], + visibility = ["//visibility:public"], +) +sh_binary( + name = "binary", + srcs = ["binary.sh"], + deps = [ + "//pkg:library2", + "@//pkg:library", + "@bazel_tools//tools/bash/runfiles", + ], +) +sh_test( + name = "test", + srcs = ["test.sh"], + deps = [ + "//pkg:library2", + "@//pkg:library", + "@bazel_tools//tools/bash/runfiles", + ], +) +EOF + + cat > other_repo/pkg/library2.sh <<'EOF' +#!/usr/bin/env bash +# --- begin runfiles.bash initialization v2 --- +# Copy-pasted from the Bazel Bash runfiles library v2. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v2 --- + +function library2() { + echo "in external/other_repo/pkg/library2.sh: '$(runfiles_current_repository)'" +} +export -f library2 +EOF + chmod +x pkg/library.sh + + cat > other_repo/pkg/binary.sh <<'EOF' +#!/usr/bin/env bash +# --- begin runfiles.bash initialization v2 --- +# Copy-pasted from the Bazel Bash runfiles library v2. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v2 --- + +echo "in external/other_repo/pkg/binary.sh: '$(runfiles_current_repository)'" +source $(rlocation _main/pkg/library.sh) +library +source $(rlocation other_repo/pkg/library2.sh) +library2 +EOF + chmod +x other_repo/pkg/binary.sh + + cat > other_repo/pkg/test.sh <<'EOF' +#!/usr/bin/env bash +# --- begin runfiles.bash initialization v2 --- +# Copy-pasted from the Bazel Bash runfiles library v2. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v2 --- + +echo "in external/other_repo/pkg/test.sh: '$(runfiles_current_repository)'" +source $(rlocation _main/pkg/library.sh) +library +source $(rlocation other_repo/pkg/library2.sh) +library2 +EOF + chmod +x other_repo/pkg/test.sh +} + +function test_bash_runfiles_current_repository_binary_enable_runfiles() { + setup_bash_runfiles_current_repository + + RUNFILES_LIB_DEBUG=1 bazel run --enable_bzlmod --enable_runfiles //pkg:binary \ + &>"$TEST_log" || fail "Run should succeed" + expect_log "in pkg/binary.sh: ''" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" + + RUNFILES_LIB_DEBUG=1 bazel run --enable_bzlmod --enable_runfiles @other_repo//pkg:binary \ + &>"$TEST_log" || fail "Run should succeed" + expect_log "in external/other_repo/pkg/binary.sh: 'other_repo'" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" +} + +function test_bash_runfiles_current_repository_test_enable_runfiles() { + setup_bash_runfiles_current_repository + + bazel test --enable_bzlmod --enable_runfiles --test_env=RUNFILES_LIB_DEBUG=1 \ + --test_output=all //pkg:test &>"$TEST_log" || fail "Test should succeed" + expect_log "in pkg/test.sh: ''" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" + + bazel test --enable_bzlmod --enable_runfiles --test_env=RUNFILES_LIB_DEBUG=1 \ + --test_output=all @other_repo//pkg:test &>"$TEST_log" || fail "Test should succeed" + expect_log "in external/other_repo/pkg/test.sh: 'other_repo'" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" +} + +function test_bash_runfiles_current_repository_binary_noenable_runfiles() { + setup_bash_runfiles_current_repository + + RUNFILES_LIB_DEBUG=1 bazel run --enable_bzlmod --noenable_runfiles //pkg:binary \ + &>"$TEST_log" || fail "Run should succeed" + expect_log "in pkg/binary.sh: ''" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" + + RUNFILES_LIB_DEBUG=1 bazel run --enable_bzlmod --noenable_runfiles @other_repo//pkg:binary \ + &>"$TEST_log" || fail "Run should succeed" + expect_log "in external/other_repo/pkg/binary.sh: 'other_repo'" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" +} + +function test_bash_runfiles_current_repository_test_noenable_runfiles() { + setup_bash_runfiles_current_repository + + bazel test --enable_bzlmod --noenable_runfiles --test_env=RUNFILES_LIB_DEBUG=1 \ + --test_output=all //pkg:test &>"$TEST_log" || fail "Test should succeed" + expect_log "in pkg/test.sh: ''" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" + + bazel test --enable_bzlmod --noenable_runfiles --test_env=RUNFILES_LIB_DEBUG=1 \ + --test_output=all @other_repo//pkg:test &>"$TEST_log" || fail "Test should succeed" + expect_log "in external/other_repo/pkg/test.sh: 'other_repo'" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" +} + +function test_bash_runfiles_current_repository_binary_nobuild_runfile_links() { + setup_bash_runfiles_current_repository + + RUNFILES_LIB_DEBUG=1 bazel run --enable_bzlmod --nobuild_runfile_links //pkg:binary \ + &>"$TEST_log" || fail "Run should succeed" + expect_log "in pkg/binary.sh: ''" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" + + RUNFILES_LIB_DEBUG=1 bazel run --enable_bzlmod --nobuild_runfile_links @other_repo//pkg:binary \ + &>"$TEST_log" || fail "Run should succeed" + expect_log "in external/other_repo/pkg/binary.sh: 'other_repo'" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" +} + +function test_bash_runfiles_current_repository_test_nobuild_runfile_links() { + setup_bash_runfiles_current_repository + + bazel test --enable_bzlmod --noenable_runfiles --nobuild_runfile_links \ + --test_env=RUNFILES_LIB_DEBUG=1 --test_output=all //pkg:test \ + &>"$TEST_log" || fail "Test should succeed" + expect_log "in pkg/test.sh: ''" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" + + bazel test --enable_bzlmod --noenable_runfiles --nobuild_runfile_links \ + --test_env=RUNFILES_LIB_DEBUG=1 --test_output=all @other_repo//pkg:test \ + &>"$TEST_log" || fail "Test should succeed" + expect_log "in external/other_repo/pkg/test.sh: 'other_repo'" + expect_log "in pkg/library.sh: ''" + expect_log "in external/other_repo/pkg/library2.sh: 'other_repo'" +} + run_suite "rules test" diff --git a/tools/bash/runfiles/runfiles.bash b/tools/bash/runfiles/runfiles.bash index d0aedd41deca89..6c6ae4661aa4b4 100644 --- a/tools/bash/runfiles/runfiles.bash +++ b/tools/bash/runfiles/runfiles.bash @@ -12,9 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Runfiles lookup library for Bazel-built Bash binaries and tests, version 2. +# Runfiles lookup library for Bazel-built Bash binaries and tests, version 3. # # VERSION HISTORY: +# - version 3: Fixes a bug in the init code on macOS and makes the library aware +# of Bzlmod repository mappings. +# Features: +# - With Bzlmod enabled, rlocation now takes the repository mapping of the +# Bazel repository containing the calling script into account when +# looking up runfiles. The new, optional second argument to rlocation can +# be used to specify the canonical name of the Bazel repository to use +# instead of this default. The new runfiles_current_repository function +# can be used to obtain the canonical name of the N-th caller's Bazel +# repository. +# Fixed: +# - Sourcing a shell script that contains the init code from a shell script +# that itself contains the init code no longer fails on macOS. +# Compatibility: +# - The init script and the runfiles library are backwards and forwards +# compatible with version 2. # - version 2: Shorter init code. # Features: # - "set -euo pipefail" only at end of init code. @@ -53,7 +69,7 @@ # # # --- begin runfiles.bash initialization v2 --- # # Copy-pasted from the Bazel Bash runfiles library v2. -# set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash +# set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash # source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ # source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ # source "$0.runfiles/$f" 2>/dev/null || \ @@ -90,6 +106,10 @@ msys*|mingw*|cygwin*) esac # Prints to stdout the runtime location of a data-dependency. +# The optional second argument can be used to specify the canonical name of the +# repository whose repository mapping should be used to resolve the repository +# part of the provided path. If not specified, the repository of the caller is +# used. function rlocation() { if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then echo >&2 "INFO[runfiles.bash]: rlocation($1): start" @@ -111,72 +131,40 @@ function rlocation() { "drive name" fi return 1 - else - if [[ -e "${RUNFILES_DIR:-/dev/null}/$1" ]]; then + fi + + if [[ -f "$RUNFILES_REPO_MAPPING" ]]; then + local -r target_repo_apparent_name=$(echo "$1" | cut -d / -f 1) + local -r remainder=$(echo "$1" | cut -d / -f 2-) + if [[ -n "$remainder" ]]; then + if [[ -z "${2+x}" ]]; then + local -r source_repo=$(runfiles_current_repository 2) + else + local -r source_repo=$2 + fi if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then - echo >&2 "INFO[runfiles.bash]: rlocation($1): found under RUNFILES_DIR ($RUNFILES_DIR), return" + echo >&2 "INFO[runfiles.bash]: rlocation($1): looking up canonical name for ($target_repo_apparent_name) from ($source_repo) in ($RUNFILES_REPO_MAPPING)" fi - echo "${RUNFILES_DIR}/$1" - elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + local -r target_repo=$(grep -m1 "^$source_repo,$target_repo_apparent_name," "$RUNFILES_REPO_MAPPING" | cut -d , -f 3) if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then - echo >&2 "INFO[runfiles.bash]: rlocation($1): looking in RUNFILES_MANIFEST_FILE ($RUNFILES_MANIFEST_FILE)" + echo >&2 "INFO[runfiles.bash]: rlocation($1): canonical name of target repo is ($target_repo)" fi - local -r result=$(grep -m1 "^$1 " "${RUNFILES_MANIFEST_FILE}" | cut -d ' ' -f 2-) - if [[ -z "$result" ]]; then - # If path references a runfile that lies under a directory that itself - # is a runfile, then only the directory is listed in the manifest. Look - # up all prefixes of path in the manifest and append the relative path - # from the prefix if there is a match. - local prefix="$1" - local prefix_result= - local new_prefix= - while true; do - new_prefix="${prefix%/*}" - [[ "$new_prefix" == "$prefix" ]] && break - prefix="$new_prefix" - prefix_result=$(grep -m1 "^$prefix " "${RUNFILES_MANIFEST_FILE}" | cut -d ' ' -f 2-) - [[ -z "$prefix_result" ]] && continue - local -r candidate="${prefix_result}${1#"${prefix}"}" - if [[ -e "$candidate" ]]; then - if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then - echo >&2 "INFO[runfiles.bash]: rlocation($1): found in manifest as ($candidate) via prefix ($prefix)" - fi - echo "$candidate" - return 0 - fi - # At this point, the manifest lookup of prefix has been successful, - # but the file at the relative path given by the suffix does not - # exist. We do not continue the lookup with a shorter prefix for two - # reasons: - # 1. Manifests generated by Bazel never contain a path that is a - # prefix of another path. - # 2. Runfiles libraries for other languages do not check for file - # existence and would have returned the non-existent path. It seems - # better to return no path rather than a potentially different, - # non-empty path. - break - done - if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then - echo >&2 "INFO[runfiles.bash]: rlocation($1): not found in manifest" - fi - echo "" + if [[ -n "$target_repo" ]]; then + local -r rlocation_path="$target_repo/$remainder" else - if [[ -e "$result" ]]; then - if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then - echo >&2 "INFO[runfiles.bash]: rlocation($1): found in manifest as ($result)" - fi - echo "$result" - fi + local -r rlocation_path="$1" fi else - if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then - echo >&2 "ERROR[runfiles.bash]: cannot look up runfile \"$1\" " \ - "(RUNFILES_DIR=\"${RUNFILES_DIR:-}\"," \ - "RUNFILES_MANIFEST_FILE=\"${RUNFILES_MANIFEST_FILE:-}\")" - fi - return 1 + local -r rlocation_path="$1" fi + else + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: rlocation($1): not using repository mapping ($RUNFILES_REPO_MAPPING) since it does not exist" + fi + local -r rlocation_path="$1" fi + + runfiles_rlocation_checked "$rlocation_path" } export -f rlocation @@ -214,3 +202,165 @@ function runfiles_export_envvars() { fi } export -f runfiles_export_envvars + +# Returns the canonical name of the Bazel repository containing the script that +# calls this function. +# The optional argument N, which defaults to 1, can be used to return the +# canonical name of the N-th caller instead. +# +# Note: This function only works correctly with Bzlmod enabled. Without Bzlmod, +# its return value is ignored if passed to rlocation. +function runfiles_current_repository() { + local -r idx=${1:-1} + local -r caller_path="${BASH_SOURCE[$idx]}" + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: runfiles_current_repository($idx): caller's path is ($caller_path)" + fi + + local rlocation_path= + + # If the runfiles manifest exists, search for an entry with target the caller's path. + if [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + # Escape $caller_path for use in the grep regex below. Also replace \ with / since the manifest + # uses / as the path separator even on Windows. + local -r normalized_caller_path="$(echo "$caller_path" | sed 's|\\\\*|/|g')" + local -r escaped_caller_path="$(echo "$normalized_caller_path" | sed 's/[^-A-Za-z0-9_/]/\\&/g')" + rlocation_path=$(grep -m1 "^[^ ]* ${escaped_caller_path}$" "${RUNFILES_MANIFEST_FILE}" | cut -d ' ' -f 1) + if [[ -z "$rlocation_path" ]]; then + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: runfiles_current_repository($idx): ($normalized_caller_path) is not the target of an entry in the runfiles manifest ($RUNFILES_MANIFEST_FILE)" + fi + return 1 + else + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: runfiles_current_repository($idx): ($normalized_caller_path) is the target of ($rlocation_path) in the runfiles manifest" + fi + fi + fi + + # If the runfiles directory exists, check if the caller's path is of the form + # $RUNFILES_DIR/rlocation_path and if so, set $rlocation_path. + if [[ -z "$rlocation_path" && -d "${RUNFILES_DIR:-/dev/null}" ]]; then + local -r normalized_caller_path="$(echo "$caller_path" | sed 's|\\\\*|/|g')" + local -r normalized_dir="$(echo "${RUNFILES_DIR%[\/]}" | sed 's|\\\\*|/|g')" + if [[ "$normalized_caller_path" == "$normalized_dir"/* ]]; then + rlocation_path=${normalized_caller_path:${#normalized_dir}} + rlocation_path=${rlocation_path:1} + fi + if [[ -z "$rlocation_path" ]]; then + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: runfiles_current_repository($idx): ($normalized_caller_path) does not lie under the runfiles directory ($normalized_dir)" + fi + # The only shell script that is not executed from the runfiles directory (if it is populated) + # is the sh_binary entrypoint. Parse its path under the execroot, using the last match to + # allow for nested execroots (e.g. in Bazel integration tests). + local -r repository=$(echo "$normalized_caller_path" | grep -E -o '/execroot/[^/]+/bazel-out/[^/]+/bin/external/[^/]+/' | tail -1 | rev | cut -d / -f 2 | rev) + if [[ -n "$repository" ]]; then + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: runfiles_current_repository($idx): ($normalized_caller_path) lies in repository ($repository)" + fi + echo "$repository" + else + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: runfiles_current_repository($idx): ($normalized_caller_path) lies in the main repository" + fi + echo "" + fi + return 0 + else + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: runfiles_current_repository($idx): ($caller_path) has path ($rlocation_path) relative to the runfiles directory ($RUNFILES_DIR)" + fi + fi + fi + + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: runfiles_current_repository($idx): ($caller_path) corresponds to rlocation path ($rlocation_path)" + fi + # Normalize the rlocation path to be of the form repo/pkg/file. + rlocation_path=${rlocation_path#_main/external/} + rlocation_path=${rlocation_path#_main/../} + local -r repository=$(echo "$rlocation_path" | cut -d / -f 1) + if [[ "$repository" == _main ]]; then + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: runfiles_current_repository($idx): ($rlocation_path) lies in the main repository" + fi + echo "" + else + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: runfiles_current_repository($idx): ($rlocation_path) lies in repository ($repository)" + fi + echo "$repository" + fi +} +export -f runfiles_current_repository + +function runfiles_rlocation_checked() { + if [[ -e "${RUNFILES_DIR:-/dev/null}/$1" ]]; then + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: rlocation($1): found under RUNFILES_DIR ($RUNFILES_DIR), return" + fi + echo "${RUNFILES_DIR}/$1" + elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: rlocation($1): looking in RUNFILES_MANIFEST_FILE ($RUNFILES_MANIFEST_FILE)" + fi + local -r result=$(grep -m1 "^$1 " "${RUNFILES_MANIFEST_FILE}" | cut -d ' ' -f 2-) + if [[ -z "$result" ]]; then + # If path references a runfile that lies under a directory that itself + # is a runfile, then only the directory is listed in the manifest. Look + # up all prefixes of path in the manifest and append the relative path + # from the prefix if there is a match. + local prefix="$1" + local prefix_result= + local new_prefix= + while true; do + new_prefix="${prefix%/*}" + [[ "$new_prefix" == "$prefix" ]] && break + prefix="$new_prefix" + prefix_result=$(grep -m1 "^$prefix " "${RUNFILES_MANIFEST_FILE}" | cut -d ' ' -f 2-) + [[ -z "$prefix_result" ]] && continue + local -r candidate="${prefix_result}${1#"${prefix}"}" + if [[ -e "$candidate" ]]; then + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: rlocation($1): found in manifest as ($candidate) via prefix ($prefix)" + fi + echo "$candidate" + return 0 + fi + # At this point, the manifest lookup of prefix has been successful, + # but the file at the relative path given by the suffix does not + # exist. We do not continue the lookup with a shorter prefix for two + # reasons: + # 1. Manifests generated by Bazel never contain a path that is a + # prefix of another path. + # 2. Runfiles libraries for other languages do not check for file + # existence and would have returned the non-existent path. It seems + # better to return no path rather than a potentially different, + # non-empty path. + break + done + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: rlocation($1): not found in manifest" + fi + echo "" + else + if [[ -e "$result" ]]; then + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "INFO[runfiles.bash]: rlocation($1): found in manifest as ($result)" + fi + echo "$result" + fi + fi + else + if [[ "${RUNFILES_LIB_DEBUG:-}" == 1 ]]; then + echo >&2 "ERROR[runfiles.bash]: cannot look up runfile \"$1\" " \ + "(RUNFILES_DIR=\"${RUNFILES_DIR:-}\"," \ + "RUNFILES_MANIFEST_FILE=\"${RUNFILES_MANIFEST_FILE:-}\")" + fi + return 1 + fi +} +export -f runfiles_rlocation_checked + +export RUNFILES_REPO_MAPPING=$(runfiles_rlocation_checked _repo_mapping 2> /dev/null) diff --git a/tools/bash/runfiles/runfiles_test.bash b/tools/bash/runfiles/runfiles_test.bash index e532824770fa3c..7105f7fc0bd082 100755 --- a/tools/bash/runfiles/runfiles_test.bash +++ b/tools/bash/runfiles/runfiles_test.bash @@ -205,6 +205,182 @@ function test_init_directory_based_runfiles() { [[ -z "$(rlocation "c d")" ]] || fail } +function test_directory_based_runfiles_with_repo_mapping_from_main() { + local tmpdir="$(mktemp -d $TEST_TMPDIR/tmp.XXXXXXXX)" + + export RUNFILES_DIR=${tmpdir}/mock/runfiles + mkdir -p "$RUNFILES_DIR" + cat > "$RUNFILES_DIR/_repo_mapping" < "$RUNFILES_DIR/_repo_mapping" < "$tmpdir/foo.repo_mapping" < "$RUNFILES_MANIFEST_FILE" << EOF +_repo_mapping $tmpdir/foo.repo_mapping +config.json $tmpdir/config.json +protobuf~3.19.2/foo/runfile $tmpdir/protobuf~3.19.2/foo/runfile +_main/bar/runfile $tmpdir/_main/bar/runfile +protobuf~3.19.2/bar/dir $tmpdir/protobuf~3.19.2/bar/dir +EOF + source "$runfiles_lib_path" + + mkdir -p "$tmpdir/_main/bar" + touch "$tmpdir/_main/bar/runfile" + mkdir -p "$tmpdir/protobuf~3.19.2/bar/dir/de eply/nes ted" + touch "$tmpdir/protobuf~3.19.2/bar/dir/file" + touch "$tmpdir/protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le" + mkdir -p "$tmpdir/protobuf~3.19.2/foo" + touch "$tmpdir/protobuf~3.19.2/foo/runfile" + touch "$tmpdir/config.json" + + [[ "$(rlocation "my_module/bar/runfile" "")" == "$tmpdir/_main/bar/runfile" ]] || fail + [[ "$(rlocation "my_workspace/bar/runfile" "")" == "$tmpdir/_main/bar/runfile" ]] || fail + [[ "$(rlocation "my_protobuf/foo/runfile" "")" == "$tmpdir/protobuf~3.19.2/foo/runfile" ]] || fail + [[ "$(rlocation "my_protobuf/bar/dir" "")" == "$tmpdir/protobuf~3.19.2/bar/dir" ]] || fail + [[ "$(rlocation "my_protobuf/bar/dir/file" "")" == "$tmpdir/protobuf~3.19.2/bar/dir/file" ]] || fail + [[ "$(rlocation "my_protobuf/bar/dir/de eply/nes ted/fi~le" "")" == "$tmpdir/protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le" ]] || fail + + [[ -z "$(rlocation "protobuf/foo/runfile" "")" ]] || fail + [[ -z "$(rlocation "protobuf/bar/dir/dir/de eply/nes ted/fi~le" "")" ]] || fail + + [[ "$(rlocation "_main/bar/runfile" "")" == "$tmpdir/_main/bar/runfile" ]] || fail + [[ "$(rlocation "protobuf~3.19.2/foo/runfile" "")" == "$tmpdir/protobuf~3.19.2/foo/runfile" ]] || fail + [[ "$(rlocation "protobuf~3.19.2/bar/dir" "")" == "$tmpdir/protobuf~3.19.2/bar/dir" ]] || fail + [[ "$(rlocation "protobuf~3.19.2/bar/dir/file" "")" == "$tmpdir/protobuf~3.19.2/bar/dir/file" ]] || fail + [[ "$(rlocation "protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le" "")" == "$tmpdir/protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le" ]] || fail + + [[ "$(rlocation "config.json" "")" == "$tmpdir/config.json" ]] || fail +} + +function test_manifest_based_runfiles_with_repo_mapping_from_other_repo() { + local tmpdir="$(mktemp -d $TEST_TMPDIR/tmp.XXXXXXXX)" + + cat > "$tmpdir/foo.repo_mapping" < "$RUNFILES_MANIFEST_FILE" << EOF +_repo_mapping $tmpdir/foo.repo_mapping +config.json $tmpdir/config.json +protobuf~3.19.2/foo/runfile $tmpdir/protobuf~3.19.2/foo/runfile +_main/bar/runfile $tmpdir/_main/bar/runfile +protobuf~3.19.2/bar/dir $tmpdir/protobuf~3.19.2/bar/dir +EOF + source "$runfiles_lib_path" + + mkdir -p "$tmpdir/_main/bar" + touch "$tmpdir/_main/bar/runfile" + mkdir -p "$tmpdir/protobuf~3.19.2/bar/dir/de eply/nes ted" + touch "$tmpdir/protobuf~3.19.2/bar/dir/file" + touch "$tmpdir/protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le" + mkdir -p "$tmpdir/protobuf~3.19.2/foo" + touch "$tmpdir/protobuf~3.19.2/foo/runfile" + touch "$tmpdir/config.json" + + [[ "$(rlocation "protobuf/foo/runfile" "protobuf~3.19.2")" == "$tmpdir/protobuf~3.19.2/foo/runfile" ]] || fail + [[ "$(rlocation "protobuf/bar/dir" "protobuf~3.19.2")" == "$tmpdir/protobuf~3.19.2/bar/dir" ]] || fail + [[ "$(rlocation "protobuf/bar/dir/file" "protobuf~3.19.2")" == "$tmpdir/protobuf~3.19.2/bar/dir/file" ]] || fail + [[ "$(rlocation "protobuf/bar/dir/de eply/nes ted/fi~le" "protobuf~3.19.2")" == "$tmpdir/protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le" ]] || fail + + [[ -z "$(rlocation "my_module/bar/runfile" "protobuf~3.19.2")" ]] || fail + [[ -z "$(rlocation "my_protobuf/bar/dir/de eply/nes ted/fi~le" "protobuf~3.19.2")" ]] || fail + + [[ "$(rlocation "_main/bar/runfile" "protobuf~3.19.2")" == "$tmpdir/_main/bar/runfile" ]] || fail + [[ "$(rlocation "protobuf~3.19.2/foo/runfile" "protobuf~3.19.2")" == "$tmpdir/protobuf~3.19.2/foo/runfile" ]] || fail + [[ "$(rlocation "protobuf~3.19.2/bar/dir" "protobuf~3.19.2")" == "$tmpdir/protobuf~3.19.2/bar/dir" ]] || fail + [[ "$(rlocation "protobuf~3.19.2/bar/dir/file" "protobuf~3.19.2")" == "$tmpdir/protobuf~3.19.2/bar/dir/file" ]] || fail + [[ "$(rlocation "protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le" "protobuf~3.19.2")" == "$tmpdir/protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le" ]] || fail + + [[ "$(rlocation "config.json" "protobuf~3.19.2")" == "$tmpdir/config.json" ]] || fail +} + function test_directory_based_envvars() { export RUNFILES_DIR=mock/runfiles export RUNFILES_MANIFEST_FILE=