diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1c6f2fd5a491b..34aa17d52bbb9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -158,6 +158,9 @@ jobs:
         with:
           submodules: true
       - name: "Building with Bazel"
+        env:
+          # Don't swap the condition! 0 is falsey and then it would always be 1
+          IREE_BAZEL_WRITE_REMOTE_CACHE: ${{ github.event_name != 'pull_request' && 1 || 0 }}
         # This doesn't really need everything in the frontends image, but we
         # want the cache to be shared with the integrations build (no point
         # building LLVM twice) and the cache key is the docker container it's
@@ -194,6 +197,8 @@ jobs:
         id: build
         env:
           IREE_TF_BINARIES_OUTPUT_DIR: iree-tf-binaries
+          # Don't swap the condition! 0 is falsey and then it would always be 1
+          IREE_BAZEL_WRITE_REMOTE_CACHE: ${{ github.event_name != 'pull_request' && 1 || 0 }}
         run: |
           ./build_tools/github_actions/docker_run.sh \
             --env "IREE_TF_BINARIES_OUTPUT_DIR=$(realpath "${IREE_TF_BINARIES_OUTPUT_DIR}")" \
diff --git a/build_tools/bazel/build_core.sh b/build_tools/bazel/build_core.sh
index 6d8fb372c1d72..a5d2666fffae4 100755
--- a/build_tools/bazel/build_core.sh
+++ b/build_tools/bazel/build_core.sh
@@ -20,7 +20,16 @@
 #   See https://docs.bazel.build/versions/master/command-line-reference.html#flag--test_tag_filters)
 #   Default: If IREE_VULKAN_DISABLE=1, "-nokokoro,-driver=vulkan". Else "-nokokoro".
 
-set -exuo pipefail
+set -xeuo pipefail
+
+
+IREE_BAZEL_READ_REMOTE_CACHE="${IREE_BAZEL_READ_REMOTE_CACHE:-1}"
+IREE_BAZEL_WRITE_REMOTE_CACHE="${IREE_BAZEL_WRITE_REMOTE_CACHE:-0}"
+BAZEL_BIN="${BAZEL_BIN:-$(which bazel)}"
+
+if (( ${IREE_BAZEL_WRITE_REMOTE_CACHE} == 1 && ${IREE_BAZEL_READ_REMOTE_CACHE} != 1 )); then
+  echo "Can't have 'IREE_BAZEL_WRITE_REMOTE_CACHE' (${IREE_BAZEL_WRITE_REMOTE_CACHE}) set without 'IREE_BAZEL_READ_REMOTE_CACHE' (${IREE_BAZEL_READ_REMOTE_CACHE})"
+fi
 
 # Use user-environment variables if set, otherwise use CI-friendly defaults.
 if ! [[ -v IREE_VULKAN_DISABLE ]]; then
@@ -64,8 +73,7 @@ if ! [[ -v TEST_TAG_FILTERS ]]; then
   TEST_TAG_FILTERS="$(IFS="," ; echo "${default_test_tag_filters[*]?}")"
 fi
 
-# Build and test everything in supported directories not excluded by the tag
-# filters.
+# Build and test everything in the main workspace (not integrations).
 # Note that somewhat contrary to its name `bazel test` will also build
 # any non-test targets specified.
 # We use `bazel query //...` piped to `bazel test` rather than the simpler
@@ -77,21 +85,37 @@ fi
 # xargs is set to high arg limits to avoid multiple Bazel invocations and will
 # hard fail if the limits are exceeded.
 # See https://github.com/bazelbuild/bazel/issues/12479
-bazel \
-  --noworkspace_rc \
-  --bazelrc=build_tools/bazel/iree.bazelrc \
-  query //... | \
-      xargs --max-args 1000000 --max-chars 1000000 --exit \
-        bazel \
-          --noworkspace_rc \
-          --bazelrc=build_tools/bazel/iree.bazelrc \
-            test \
-              --color=yes \
-              ${test_env_args[@]} \
-              --config=generic_clang \
-              --build_tag_filters="${BUILD_TAG_FILTERS?}" \
-              --test_tag_filters="${TEST_TAG_FILTERS?}" \
-              --keep_going \
-              --test_output=errors \
-              --config=rs \
-              --config=remote_cache_bazel_ci
+
+declare -a BAZEL_STARTUP_CMD=(
+  "${BAZEL_BIN}"
+  --noworkspace_rc
+  --bazelrc=build_tools/bazel/iree.bazelrc
+)
+
+declare -a BAZEL_TEST_CMD=(
+  "${BAZEL_STARTUP_CMD[@]}"
+  test
+)
+
+if (( IREE_BAZEL_READ_REMOTE_CACHE == 1 )); then
+  BAZEL_TEST_CMD+=(--config=remote_cache_bazel_ci)
+fi
+
+if (( IREE_BAZEL_WRITE_REMOTE_CACHE != 1 )); then
+  BAZEL_TEST_CMD+=(--noremote_upload_local_results)
+fi
+
+BAZEL_TEST_CMD+=(
+  --color=yes
+  "${test_env_args[@]}"
+  --build_tag_filters="${BUILD_TAG_FILTERS?}"
+  --test_tag_filters="${TEST_TAG_FILTERS?}"
+  --keep_going
+  --test_output=errors
+  --config=rs
+  --config=generic_clang
+)
+
+"${BAZEL_STARTUP_CMD[@]}" query //... | \
+  xargs --max-args 1000000 --max-chars 1000000 --exit \
+    "${BAZEL_TEST_CMD[@]}"
diff --git a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-swiftshader/core/build.sh b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-swiftshader/core/build.sh
index 68d89a699cbbd..2e7f6def8514e 100755
--- a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-swiftshader/core/build.sh
+++ b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-swiftshader/core/build.sh
@@ -24,5 +24,7 @@ bazel --version
 echo "Initializing submodules"
 git submodule update --init --jobs 8 --depth 1
 
+export IREE_BAZEL_WRITE_REMOTE_CACHE=1
+
 echo "Building and testing with bazel"
 ./build_tools/bazel/build_core.sh