Introduce a native pants client.

Currently the client only handles talking to a pantsd brought up by
other means. The Pants repo ./pants script is updated to optionally use
the native client and prop up pantsd using the python client as needed.

Work towards pantsbuild#11831

# Building wheels and fs_util will be skipped. Delete if not intended.
[ci skip-build-wheels]

.gitignore

@@ -54,3 +54,4 @@ GRTAGS
 GSYMS
 GTAGS
 .mypy_cache/
+/.pants

build-support/bin/rust/bootstrap_code.sh

@@ -30,13 +30,19 @@ esac
 readonly NATIVE_ENGINE_BINARY="native_engine.so"
 readonly NATIVE_ENGINE_RESOURCE="${REPO_ROOT}/src/python/pants/engine/internals/${NATIVE_ENGINE_BINARY}"
 readonly NATIVE_ENGINE_RESOURCE_METADATA="${NATIVE_ENGINE_RESOURCE}.metadata"
+readonly NATIVE_CLIENT_PATH="${REPO_ROOT}/.pants"
 
-function _build_native_code() {
+function _build_native_engine() {
   # NB: See Cargo.toml with regard to the `extension-module` feature.
   "${REPO_ROOT}/cargo" build --features=extension-module ${MODE_FLAG} -p engine || die
   echo "${NATIVE_ROOT}/target/${MODE}/libengine.${LIB_EXTENSION}"
 }
 
+function _build_native_client() {
+  "${REPO_ROOT}/cargo" build ${MODE_FLAG} -p client || die
+  echo "${NATIVE_ROOT}/target/${MODE}/pants"
+}
+
 function bootstrap_native_code() {
   # We expose a safety valve to skip compilation iff the user already has `native_engine.so`. This
   # can result in using a stale `native_engine.so`, but we trust that the user knows what
@@ -56,24 +62,34 @@ function bootstrap_native_code() {
   if [[ -f "${NATIVE_ENGINE_RESOURCE_METADATA}" ]]; then
     engine_version_in_metadata="$(sed -n 's/^engine_version: //p' "${NATIVE_ENGINE_RESOURCE_METADATA}")"
   fi
-  if [[ ! -f "${NATIVE_ENGINE_RESOURCE}" || "${engine_version_calculated}" != "${engine_version_in_metadata}" ]]; then
+  if [[ ! -f "${NATIVE_ENGINE_RESOURCE}" || ! -f "${NATIVE_CLIENT_PATH}" || \
+    "${engine_version_calculated}" != "${engine_version_in_metadata}" ]]; then
+
     echo "Building native engine"
-    local -r native_binary="$(_build_native_code)"
+    local -r native_engine="$(_build_native_engine)"
 
-    # If bootstrapping the native engine fails, don't attempt to run pants
-    # afterwards.
-    if [[ ! -f "${native_binary}" ]]; then
+    # If bootstrapping the native engine fails, don't attempt to run pants afterwards.
+    if [[ ! -f "${native_engine}" ]]; then
       die "Failed to build native engine."
     fi
 
+    echo "Building native client"
+    local -r native_client="$(_build_native_client)"
+
+    # If bootstrapping the native client fails, don't attempt to run pants afterwards.
+    if [[ ! -f "${native_client}" ]]; then
+      die "Failed to build native client."
+    fi
+
     # Pick up Cargo.lock changes if any caused by the `cargo build`.
     engine_version_calculated="$(calculate_current_hash)"
 
     # Create the native engine resource.
     # NB: On Mac Silicon, for some reason, first removing the old native_engine.so is necessary to avoid the Pants
     #  process from being killed when recompiling.
-    rm -f "${NATIVE_ENGINE_RESOURCE}"
-    cp "${native_binary}" "${NATIVE_ENGINE_RESOURCE}"
+    rm -f "${NATIVE_ENGINE_RESOURCE}" "${NATIVE_CLIENT_PATH}"
+    cp "${native_engine}" "${NATIVE_ENGINE_RESOURCE}"
+    cp "${native_client}" "${NATIVE_CLIENT_PATH}"
 
     # Create the accompanying metadata file.
     local -r metadata_file=$(mktemp -t pants.native_engine.metadata.XXXXXX)

pants

@@ -38,18 +38,32 @@ source "${HERE}/build-support/pants_venv"
 source "${HERE}/build-support/bin/rust/bootstrap_code.sh"
 
 function exec_pants_bare() {
-
-  PANTS_EXE="${HERE}/src/python/pants/bin/pants_loader.py"
+  PANTS_NATIVE_EXE="${HERE}/.pants"
+  PANTS_PY_EXE="${HERE}/src/python/pants/bin/pants_loader.py"
   PANTS_SRCPATH="${HERE}/src/python"
 
   # Redirect activation and native bootstrap to ensure that they don't interfere with stdout.
   activate_pants_venv 1>&2
   bootstrap_native_code 1>&2
 
+  if [ -n "${USE_NATIVE_PANTS}" ]; then
+    set +e
+    "${PANTS_NATIVE_EXE}" "$@"
+    result=$?
+    # N.B.: The native pants client currently relies on pantsd being up. If it's not, it will fail
+    # with exit code 75 (EX_TEMPFAIL in /usr/include/sysexits.h) and we should fall through to the
+    # python pants client which knows how to start up pantsd. This failure takes O(1ms); so has no
+    # appreciable impact on --no-pantsd runs.
+    if ((result != 75)); then
+      exit ${result}
+    fi
+    set -e
+  fi
+
   # Because the venv has been activated, we simply say `python` and it will use the venv's version. We cannot use
   # `${PY}` because it could use the wrong interpreter if the value is an absolute path.
   PYTHONPATH="${PANTS_SRCPATH}:${PYTHONPATH}" RUNNING_PANTS_FROM_SOURCES=1 \
-    exec python "${PANTS_EXE}" "$@"
+    exec python "${PANTS_PY_EXE}" "$@"
 }
 
 exec_pants_bare "$@"