Place Docker workdir in consistent location (#10327)

Ironically, I recently did the opposite in our release builds when
trying to fix some path recognition stuff in
#10126. It would be nice if that
shared infra with everything else :-/

.github/workflows/ci.yml

@@ -342,7 +342,7 @@ jobs:
         run: |
           ./build_tools/github_actions/docker_run.sh \
             --env "IREE_BAZEL_WRITE_REMOTE_CACHE=${IREE_BAZEL_WRITE_REMOTE_CACHE}" \
-            --env "IREE_TF_BINARIES_OUTPUT_DIR=$(realpath "${IREE_TF_BINARIES_OUTPUT_DIR}")" \
+            --env "IREE_TF_BINARIES_OUTPUT_DIR=${IREE_TF_BINARIES_OUTPUT_DIR}" \
             gcr.io/iree-oss/frontends-swiftshader@sha256:3090418a8d8a64c356d35eff285af32570a72f41127aa123209c1562f57abb01 \
             build_tools/cmake/build_tf_binaries.sh
           echo "::set-output name=binaries-dir::${IREE_TF_BINARIES_OUTPUT_DIR}"
@@ -387,10 +387,10 @@ jobs:
       - name: "Downloading TF binaries archive"
         run: gcloud alpha storage cp "${TF_BINARIES_GCS_ARTIFACT}" "${TF_BINARIES_ARCHIVE}"
       - name: "Extracting TF binaries archive"
-        run: tar -xf "${TF_BINARIES_ARCHIVE}"
+        run: tar -xvf "${TF_BINARIES_ARCHIVE}"
       - name: "Symlinking TF binaries"
         run: |
-          ./integrations/tensorflow/symlink_binaries.sh "$(realpath "${TF_BINARIES_DIR}")"
+          ./integrations/tensorflow/symlink_binaries.sh "${TF_BINARIES_DIR}"
       - name: "Downloading build dir archive"
         run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
       - name: "Extracting build dir archive"
@@ -426,10 +426,10 @@ jobs:
       - name: "Downloading TF binaries archive"
         run: gcloud alpha storage cp "${TF_BINARIES_GCS_ARTIFACT}" "${TF_BINARIES_ARCHIVE}"
       - name: "Extracting TF binaries archive"
-        run: tar -xf "${TF_BINARIES_ARCHIVE}"
+        run: tar -xvf "${TF_BINARIES_ARCHIVE}"
       - name: "Symlinking TF binaries"
         run: |
-          ./integrations/tensorflow/symlink_binaries.sh "$(realpath "${TF_BINARIES_DIR}")"
+          ./integrations/tensorflow/symlink_binaries.sh "${TF_BINARIES_DIR}"
       - name: "Downloading build dir archive"
         run: gcloud alpha storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}"
       - name: "Extracting build dir archive"

build_tools/cmake/build_tf_binaries.sh

@@ -18,12 +18,15 @@ IREE_USE_WORKSPACE_RC="${IREE_USE_WORKSPACE_RC:-0}"
 IREE_BAZEL_READ_REMOTE_CACHE="${IREE_BAZEL_READ_REMOTE_CACHE:-1}"
 IREE_BAZEL_WRITE_REMOTE_CACHE="${IREE_BAZEL_WRITE_REMOTE_CACHE:-0}"
 IREE_TF_BINARIES_OUTPUT_DIR="${IREE_TF_BINARIES_OUTPUT_DIR:-}"
+INTEGRATIONS_DIR="${ROOT_DIR}/integrations/tensorflow"
 
 if (( ${IREE_BAZEL_WRITE_REMOTE_CACHE} == 1 && ${IREE_BAZEL_READ_REMOTE_CACHE} != 1 )); then
   echo "Can't have 'IREE_BAZEL_WRITE_REMOTE_CACHE' (${IREE_BAZEL_WRITE_REMOTE_CACHE}) set without 'IREE_BAZEL_READ_REMOTE_CACHE' (${IREE_BAZEL_READ_REMOTE_CACHE})"
 fi
 
-cd "${ROOT_DIR}/integrations/tensorflow"
+# We want to get back to wherever we were called from and output to the output
+# directory relative to that.
+pushd "${INTEGRATIONS_DIR}" > /dev/null
 
 BAZEL_BIN=${BAZEL_BIN:-$(which bazel)}
 
@@ -61,12 +64,13 @@ BAZEL_TEST_CMD+=(
    xargs --max-args 1000000 --max-chars 1000000 --exit \
     "${BAZEL_TEST_CMD[@]}"
 
+popd > /dev/null
 
 if [[ "${IREE_TF_BINARIES_OUTPUT_DIR}" != "" ]]; then
   mkdir -p "${IREE_TF_BINARIES_OUTPUT_DIR}"
   cp \
-    bazel-bin/iree_tf_compiler/iree-import-tf \
-    bazel-bin/iree_tf_compiler/iree-import-tflite \
-    bazel-bin/iree_tf_compiler/iree-import-xla \
+    "${INTEGRATIONS_DIR}/bazel-bin/iree_tf_compiler/iree-import-tf" \
+    "${INTEGRATIONS_DIR}/bazel-bin/iree_tf_compiler/iree-import-tflite" \
+    "${INTEGRATIONS_DIR}/bazel-bin/iree_tf_compiler/iree-import-xla" \
     "${IREE_TF_BINARIES_OUTPUT_DIR}"
 fi

build_tools/docker/docker_run.sh

@@ -6,17 +6,24 @@
 
 set -euo pipefail
 
+# It's convenient to have the paths inside the container match the paths
+# outside. This creates an issue, however, because we pass around CMake build
+# directories, which use absolute paths, so it's important that the paths match
+# between runners. Doing things this way allows runners to change their working
+# directory and enables local reproduction of issues.
+DOCKER_CONTAINER_WORKDIR="${DOCKER_CONTAINER_WORKDIR:-/work}"
+
 # Sets up files and environment to enable running scripts in docker.
 # In particular, does some shenanigans to enable running with the current user.
 # Some of this setup is only strictly necessary for Bazel, but it doesn't hurt
 # for anything else.
-# Requires that DOCKER_WORKDIR and DOCKER_TMPDIR have been set
+# Requires that DOCKER_HOST_WORKDIR and DOCKER_HOST_TMPDIR have been set
 function docker_run() {
     # Make the source repository available and launch containers in that
     # directory.
     DOCKER_RUN_ARGS=(
-      --volume="${DOCKER_WORKDIR}:${DOCKER_WORKDIR}"
-      --workdir="${DOCKER_WORKDIR}"
+      --volume="${DOCKER_HOST_WORKDIR}:${DOCKER_CONTAINER_WORKDIR}"
+      --workdir="${DOCKER_CONTAINER_WORKDIR}"
     )
 
     # Delete the container after the run is complete.
@@ -45,7 +52,7 @@ function docker_run() {
     # such that they don't contain the information about normal users and we
     # want these scripts to be runnable locally for debugging.
     # Instead we dump the results of `getent` to some fake files.
-    local fake_etc_dir="${DOCKER_TMPDIR}/fake_etc"
+    local fake_etc_dir="${DOCKER_HOST_TMPDIR}/fake_etc"
     mkdir -p "${fake_etc_dir?}"
 
     local fake_group="${fake_etc_dir?}/group"
@@ -73,7 +80,7 @@ function docker_run() {
     #      the difference between a persistent SSD and a local scratch SSD can
     #      be huge. In particular, Kokoro has the home directory on the former
     #      and the work directory on the latter.
-    local fake_home_dir="${DOCKER_TMPDIR}/fake_home"
+    local fake_home_dir="${DOCKER_HOST_TMPDIR}/fake_home"
     mkdir -p "${fake_home_dir}"
 
     DOCKER_RUN_ARGS+=(

build_tools/github_actions/docker_run.sh

@@ -10,7 +10,7 @@
 
 set -euo pipefail
 
-export DOCKER_WORKDIR="${GITHUB_WORKSPACE}"
-export DOCKER_TMPDIR="${RUNNER_TEMP}"
+export DOCKER_HOST_WORKDIR="${GITHUB_WORKSPACE}"
+export DOCKER_HOST_TMPDIR="${RUNNER_TEMP}"
 
 "${GITHUB_WORKSPACE}/build_tools/docker/docker_run.sh" "$@"

integrations/tensorflow/symlink_binaries.sh

@@ -4,21 +4,29 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# Symlinks built binaries from the bazel-bin/ directory into the corresponding
-# python packages.
+# Symlinks built TF import binaries from the specified directory (defaults to
+# the appropriate bazel-bin/ subdirectory) into the corresponding Python
+# packages. If the binary directory is contained within the root directory, it
+# uses a relative symlink, which makes this work when the repository is copied
+# or mounted in a Docker container under some other path.
 
 set -euo pipefail
 
-cd "$(dirname $0)"
+ROOT_DIR="${ROOT_DIR:-$(git rev-parse --show-toplevel)}"
+SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")";
 
-BINARIES_DIR="${1:-${PWD}/bazel-bin/iree_tf_compiler}"
+BINARIES_DIR="${1:-${SCRIPT_DIR}/bazel-bin/iree_tf_compiler}"
 
-if [ -f "${BINARIES_DIR}/iree-import-tf" ]; then
-  ln -sf "${BINARIES_DIR}/iree-import-tf" python_projects/iree_tf/iree/tools/tf/
-fi
-if [ -f "${BINARIES_DIR}/iree-import-tflite" ]; then
-  ln -sf "${BINARIES_DIR}/iree-import-tflite" python_projects/iree_tflite/iree/tools/tflite/
-fi
-if [ -f "${BINARIES_DIR}/iree-import-xla" ]; then
-  ln -sf "${BINARIES_DIR}/iree-import-xla" python_projects/iree_xla/iree/tools/xla/
-fi
+function symlink_import_binary() {
+  local type="$1"
+  local import_binary="${BINARIES_DIR}/iree-import-${type}"
+  if [ -f "${import_binary}" ]; then
+    local to="${SCRIPT_DIR}/python_projects/iree_${type}/iree/tools/${type}"
+    local from="$(realpath --no-symlinks --relative-to=${to} --relative-base="${ROOT_DIR}" "${import_binary}")"
+    ln --symbolic --verbose --force "${from}" "${to}"
+  fi
+}
+
+symlink_import_binary tf
+symlink_import_binary tflite
+symlink_import_binary xla