From b00ccf9f66dab469ea12b4f23ac8a91609074057 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 7 Apr 2021 08:31:44 -0700 Subject: [PATCH] Introduce a native pants client. Currently the client only handles talking to a pantsd brought up by other means. The Pants repo ./pants script is updated to optionally use the native client and prop up pantsd using the python client as needed. Work towards #11831 # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .gitignore | 1 + build-support/bin/rust/bootstrap_code.sh | 32 +- pants | 20 +- src/rust/engine/Cargo.lock | 412 ++++++++++++++++--- src/rust/engine/Cargo.toml | 2 + src/rust/engine/client/Cargo.toml | 31 ++ src/rust/engine/client/src/build_root.rs | 165 ++++++++ src/rust/engine/client/src/client.rs | 128 ++++++ src/rust/engine/client/src/lib.rs | 44 ++ src/rust/engine/client/src/main.rs | 155 +++++++ src/rust/engine/client/src/options/args.rs | 90 ++++ src/rust/engine/client/src/options/config.rs | 197 +++++++++ src/rust/engine/client/src/options/env.rs | 53 +++ src/rust/engine/client/src/options/id.rs | 180 ++++++++ src/rust/engine/client/src/options/mod.rs | 223 ++++++++++ src/rust/engine/client/src/options/parse.rs | 152 +++++++ src/rust/engine/client/src/pantsd.rs | 247 +++++++++++ 17 files changed, 2059 insertions(+), 73 deletions(-) create mode 100644 src/rust/engine/client/Cargo.toml create mode 100644 src/rust/engine/client/src/build_root.rs create mode 100644 src/rust/engine/client/src/client.rs create mode 100644 src/rust/engine/client/src/lib.rs create mode 100644 src/rust/engine/client/src/main.rs create mode 100644 src/rust/engine/client/src/options/args.rs create mode 100644 src/rust/engine/client/src/options/config.rs create mode 100644 src/rust/engine/client/src/options/env.rs create mode 100644 src/rust/engine/client/src/options/id.rs create mode 100644 src/rust/engine/client/src/options/mod.rs create mode 100644 src/rust/engine/client/src/options/parse.rs create mode 100644 src/rust/engine/client/src/pantsd.rs diff --git a/.gitignore b/.gitignore index 9976747ca576..eb72ae612086 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,4 @@ GRTAGS GSYMS GTAGS .mypy_cache/ +/.pants diff --git a/build-support/bin/rust/bootstrap_code.sh b/build-support/bin/rust/bootstrap_code.sh index 4d020f63a624..bb2023cdffe6 100644 --- a/build-support/bin/rust/bootstrap_code.sh +++ b/build-support/bin/rust/bootstrap_code.sh @@ -30,13 +30,19 @@ esac readonly NATIVE_ENGINE_BINARY="native_engine.so" readonly NATIVE_ENGINE_RESOURCE="${REPO_ROOT}/src/python/pants/engine/internals/${NATIVE_ENGINE_BINARY}" readonly NATIVE_ENGINE_RESOURCE_METADATA="${NATIVE_ENGINE_RESOURCE}.metadata" +readonly NATIVE_CLIENT_PATH="${REPO_ROOT}/.pants" -function _build_native_code() { +function _build_native_engine() { # NB: See Cargo.toml with regard to the `extension-module` feature. "${REPO_ROOT}/cargo" build --features=extension-module ${MODE_FLAG} -p engine || die echo "${NATIVE_ROOT}/target/${MODE}/libengine.${LIB_EXTENSION}" } +function _build_native_client() { + "${REPO_ROOT}/cargo" build ${MODE_FLAG} -p client || die + echo "${NATIVE_ROOT}/target/${MODE}/pants" +} + function bootstrap_native_code() { # We expose a safety valve to skip compilation iff the user already has `native_engine.so`. This # can result in using a stale `native_engine.so`, but we trust that the user knows what @@ -56,24 +62,34 @@ function bootstrap_native_code() { if [[ -f "${NATIVE_ENGINE_RESOURCE_METADATA}" ]]; then engine_version_in_metadata="$(sed -n 's/^engine_version: //p' "${NATIVE_ENGINE_RESOURCE_METADATA}")" fi - if [[ ! -f "${NATIVE_ENGINE_RESOURCE}" || "${engine_version_calculated}" != "${engine_version_in_metadata}" ]]; then + if [[ ! -f "${NATIVE_ENGINE_RESOURCE}" || ! -f "${NATIVE_CLIENT_PATH}" || \ + "${engine_version_calculated}" != "${engine_version_in_metadata}" ]]; then + echo "Building native engine" - local -r native_binary="$(_build_native_code)" + local -r native_engine="$(_build_native_engine)" - # If bootstrapping the native engine fails, don't attempt to run pants - # afterwards. - if [[ ! -f "${native_binary}" ]]; then + # If bootstrapping the native engine fails, don't attempt to run pants afterwards. + if [[ ! -f "${native_engine}" ]]; then die "Failed to build native engine." fi + echo "Building native client" + local -r native_client="$(_build_native_client)" + + # If bootstrapping the native client fails, don't attempt to run pants afterwards. + if [[ ! -f "${native_client}" ]]; then + die "Failed to build native client." + fi + # Pick up Cargo.lock changes if any caused by the `cargo build`. engine_version_calculated="$(calculate_current_hash)" # Create the native engine resource. # NB: On Mac Silicon, for some reason, first removing the old native_engine.so is necessary to avoid the Pants # process from being killed when recompiling. - rm -f "${NATIVE_ENGINE_RESOURCE}" - cp "${native_binary}" "${NATIVE_ENGINE_RESOURCE}" + rm -f "${NATIVE_ENGINE_RESOURCE}" "${NATIVE_CLIENT_PATH}" + cp "${native_engine}" "${NATIVE_ENGINE_RESOURCE}" + cp "${native_client}" "${NATIVE_CLIENT_PATH}" # Create the accompanying metadata file. local -r metadata_file=$(mktemp -t pants.native_engine.metadata.XXXXXX) diff --git a/pants b/pants index 2a0a4b481fbf..6cbbcc920434 100755 --- a/pants +++ b/pants @@ -38,18 +38,32 @@ source "${HERE}/build-support/pants_venv" source "${HERE}/build-support/bin/rust/bootstrap_code.sh" function exec_pants_bare() { - - PANTS_EXE="${HERE}/src/python/pants/bin/pants_loader.py" + PANTS_NATIVE_EXE="${HERE}/.pants" + PANTS_PY_EXE="${HERE}/src/python/pants/bin/pants_loader.py" PANTS_SRCPATH="${HERE}/src/python" # Redirect activation and native bootstrap to ensure that they don't interfere with stdout. activate_pants_venv 1>&2 bootstrap_native_code 1>&2 + if [ -n "${USE_NATIVE_PANTS}" ]; then + set +e + "${PANTS_NATIVE_EXE}" "$@" + result=$? + # N.B.: The native pants client currently relies on pantsd being up. If it's not, it will fail + # with exit code 75 (EX_TEMPFAIL in /usr/include/sysexits.h) and we should fall through to the + # python pants client which knows how to start up pantsd. This failure takes O(1ms); so has no + # appreciable impact on --no-pantsd runs. + if ((result != 75)); then + exit ${result} + fi + set -e + fi + # Because the venv has been activated, we simply say `python` and it will use the venv's version. We cannot use # `${PY}` because it could use the wrong interpreter if the value is an absolute path. PYTHONPATH="${PANTS_SRCPATH}:${PYTHONPATH}" RUNNING_PANTS_FROM_SOURCES=1 \ - exec python "${PANTS_EXE}" "$@" + exec python "${PANTS_PY_EXE}" "$@" } exec_pants_bare "$@" diff --git a/src/rust/engine/Cargo.lock b/src/rust/engine/Cargo.lock index 7561c3d5851a..b00a4cac824e 100644 --- a/src/rust/engine/Cargo.lock +++ b/src/rust/engine/Cargo.lock @@ -1,5 +1,19 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "addr2line" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" +dependencies = [ + "cpp_demangle", + "fallible-iterator", + "gimli", + "object", + "rustc-demangle", + "smallvec 1.6.1", +] + [[package]] name = "adler" version = "0.2.3" @@ -26,9 +40,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.36" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68803225a7b13e47191bab76f2687382b60d259e8cf37f6e1893658b84bb9479" +checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" [[package]] name = "anymap" @@ -151,6 +165,19 @@ dependencies = [ "walkdir 2.3.1", ] +[[package]] +name = "benfred-read-process-memory" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ba08144670324a93f7b6502ff6ca679bfd1efa5230c4efff49801714b3854a" +dependencies = [ + "kernel32-sys", + "libc", + "log 0.4.14", + "mach", + "winapi 0.2.8", +] + [[package]] name = "bincode" version = "1.3.1" @@ -202,7 +229,7 @@ dependencies = [ "futures", "hashing", "libc", - "log 0.4.11", + "log 0.4.14", "parking_lot", "protobuf", "store", @@ -318,6 +345,29 @@ dependencies = [ "vec_map", ] +[[package]] +name = "client" +version = "0.0.1" +dependencies = [ + "env_logger", + "futures", + "lazy_static", + "libc", + "log 0.4.14", + "nails", + "nix 0.20.0", + "peg", + "remoteprocess", + "sha2", + "shellexpand", + "strum", + "strum_macros", + "tempdir", + "tokio", + "toml", + "uname", +] + [[package]] name = "cloudabi" version = "0.0.3" @@ -342,7 +392,7 @@ dependencies = [ name = "concrete_time" version = "0.0.1" dependencies = [ - "log 0.4.11", + "log 0.4.14", "prost", "prost-types", "serde", @@ -394,6 +444,16 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +[[package]] +name = "cpp_demangle" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44919ecaf6f99e8e737bc239408931c9a01e9a6c74814fee8242dd2506b65390" +dependencies = [ + "cfg-if 1.0.0", + "glob 0.3.0", +] + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -649,7 +709,7 @@ dependencies = [ "itertools 0.8.2", "lazy_static", "libc", - "log 0.4.11", + "log 0.4.14", "logging", "mock", "nailgun", @@ -686,7 +746,7 @@ checksum = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" dependencies = [ "atty", "humantime", - "log 0.4.11", + "log 0.4.14", "regex", "termcolor", ] @@ -712,6 +772,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "filetime" version = "0.2.13" @@ -772,10 +838,10 @@ dependencies = [ "bytes 1.0.1", "dirs-next", "futures", - "glob", + "glob 0.2.11", "ignore", "lazy_static", - "log 0.4.11", + "log 0.4.14", "parking_lot", "rlimit", "task_executor", @@ -870,9 +936,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253" dependencies = [ "futures-channel", "futures-core", @@ -885,9 +951,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" +checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" dependencies = [ "futures-core", "futures-sink", @@ -895,15 +961,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" +checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" [[package]] name = "futures-executor" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" dependencies = [ "futures-core", "futures-task", @@ -912,15 +978,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" +checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" [[package]] name = "futures-macro" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" dependencies = [ "proc-macro-hack", "proc-macro2 1.0.24", @@ -930,21 +996,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" +checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" [[package]] name = "futures-task" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" +checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" [[package]] name = "futures-util" -version = "0.3.8" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" dependencies = [ "futures-channel", "futures-core", @@ -953,7 +1019,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project 1.0.2", + "pin-project-lite", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -998,12 +1064,28 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] +[[package]] +name = "gimli" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" +dependencies = [ + "fallible-iterator", + "stable_deref_trait", +] + [[package]] name = "glob" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "globset" version = "0.4.6" @@ -1013,10 +1095,21 @@ dependencies = [ "aho-corasick", "bstr", "fnv", - "log 0.4.11", + "log 0.4.14", "regex", ] +[[package]] +name = "goblin" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "669cdc3826f69a51d3f8fc3f86de81c2378110254f678b8407977736122057a4" +dependencies = [ + "log 0.4.14", + "plain", + "scroll", +] + [[package]] name = "graph" version = "0.0.1" @@ -1026,7 +1119,7 @@ dependencies = [ "env_logger", "fnv", "futures", - "log 0.4.11", + "log 0.4.14", "parking_lot", "petgraph 0.4.13", "rand 0.8.2", @@ -1205,7 +1298,7 @@ checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" dependencies = [ "futures-util", "hyper", - "log 0.4.11", + "log 0.4.14", "rustls", "tokio", "tokio-rustls", @@ -1232,7 +1325,7 @@ dependencies = [ "crossbeam-utils 0.8.1", "globset", "lazy_static", - "log 0.4.11", + "log 0.4.14", "memchr", "regex", "same-file", @@ -1385,9 +1478,19 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.86" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" + +[[package]] +name = "libproc" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" +checksum = "15fb50befee2d3be15b38c93ef79ba22ecbd667874bf692309ffdff179282b8d" +dependencies = [ + "errno", + "libc", +] [[package]] name = "lmdb" @@ -1443,16 +1546,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" dependencies = [ - "log 0.4.11", + "log 0.4.14", ] [[package]] name = "log" -version = "0.4.11" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] @@ -1463,7 +1566,7 @@ dependencies = [ "chrono", "colored", "lazy_static", - "log 0.4.11", + "log 0.4.14", "num_enum", "parking_lot", "regex", @@ -1472,6 +1575,21 @@ dependencies = [ "uuid", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "mach_o_sys" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e854583a83f20cf329bb9283366335387f7db59d640d1412167e05fedb98826" + [[package]] name = "maplit" version = "1.0.2" @@ -1496,6 +1614,16 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "memoffset" version = "0.6.1" @@ -1533,7 +1661,7 @@ dependencies = [ "iovec", "kernel32-sys", "libc", - "log 0.4.11", + "log 0.4.14", "miow 0.2.2", "net2", "slab", @@ -1547,7 +1675,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e50ae3f04d169fcc9bde0b547d1c205219b7157e07ded9c5aff03e0637cb3ed7" dependencies = [ "libc", - "log 0.4.11", + "log 0.4.14", "miow 0.3.6", "ntapi", "winapi 0.3.9", @@ -1560,7 +1688,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" dependencies = [ "lazycell", - "log 0.4.11", + "log 0.4.14", "mio 0.6.23", "slab", ] @@ -1597,7 +1725,7 @@ dependencies = [ "futures", "hashing", "hyper", - "log 0.4.11", + "log 0.4.14", "parking_lot", "prost", "prost-types", @@ -1619,7 +1747,7 @@ dependencies = [ "async_latch", "bytes 1.0.1", "futures", - "log 0.4.11", + "log 0.4.14", "nails", "os_pipe", "task_executor", @@ -1635,7 +1763,7 @@ dependencies = [ "byteorder", "bytes 1.0.1", "futures", - "log 0.4.11", + "log 0.4.14", "tokio", "tokio-util", ] @@ -1651,6 +1779,30 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "nix" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", +] + +[[package]] +name = "nix" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa9b4819da1bc61c0ea48b63b7bc8604064dd43013e7cc325df098d49cd7c18a" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", +] + [[package]] name = "nom" version = "6.0.1" @@ -1807,6 +1959,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" +dependencies = [ + "flate2", + "wasmparser", +] + [[package]] name = "once_cell" version = "1.5.2" @@ -1878,6 +2040,33 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" +[[package]] +name = "peg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c0b841ea54f523f7aa556956fbd293bcbe06f2e67d2eb732b7278aaf1d166a" +dependencies = [ + "peg-macros", + "peg-runtime", +] + +[[package]] +name = "peg-macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aa52829b8decbef693af90202711348ab001456803ba2a98eb4ec8fb70844c" +dependencies = [ + "peg-runtime", + "proc-macro2 1.0.24", + "quote 1.0.8", +] + +[[package]] +name = "peg-runtime" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c719dcf55f09a3a7e764c6649ab594c18a177e3599c467983cdf644bfc0a4088" + [[package]] name = "percent-encoding" version = "2.1.0" @@ -1962,6 +2151,12 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "plotters" version = "0.2.15" @@ -2043,6 +2238,19 @@ dependencies = [ "unicode-xid 0.2.1", ] +[[package]] +name = "proc-maps" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7e49bf46401bf141cb9d2bba151efcfa85f2956e51c6794fe07a3803ec6004" +dependencies = [ + "anyhow", + "libc", + "libproc", + "mach", + "winapi 0.3.9", +] + [[package]] name = "process_execution" version = "0.0.1" @@ -2062,7 +2270,7 @@ dependencies = [ "itertools 0.8.2", "lazy_static", "libc", - "log 0.4.11", + "log 0.4.14", "maplit", "mock", "nails", @@ -2104,7 +2312,7 @@ dependencies = [ "fs", "futures", "hashing", - "log 0.4.11", + "log 0.4.14", "process_execution", "prost", "shlex", @@ -2134,7 +2342,7 @@ dependencies = [ "bytes 1.0.1", "heck", "itertools 0.9.0", - "log 0.4.11", + "log 0.4.14", "multimap", "petgraph 0.5.1", "prost", @@ -2459,6 +2667,29 @@ version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" +[[package]] +name = "remoteprocess" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0d23133e0101470829111c52d95c1fd07f4925b9ab59c99a4f9c6034a1648b6" +dependencies = [ + "addr2line", + "benfred-read-process-memory", + "goblin", + "lazy_static", + "libc", + "libproc", + "log 0.4.14", + "mach", + "mach_o_sys", + "memmap", + "nix 0.19.1", + "object", + "proc-maps", + "regex", + "winapi 0.3.9", +] + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -2486,7 +2717,7 @@ dependencies = [ "ipnet", "js-sys", "lazy_static", - "log 0.4.11", + "log 0.4.14", "mime", "percent-encoding", "pin-project-lite", @@ -2534,10 +2765,16 @@ version = "0.0.1" dependencies = [ "env_logger", "indexmap", - "log 0.4.11", + "log 0.4.14", "petgraph 0.4.13", ] +[[package]] +name = "rustc-demangle" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" + [[package]] name = "rustc-serialize" version = "0.3.24" @@ -2560,7 +2797,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" dependencies = [ "base64", - "log 0.4.11", + "log 0.4.14", "ring", "sct", "webpki", @@ -2609,6 +2846,26 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scroll" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda28d4b4830b807a8b43f7b0e6b5df875311b3e7621d84577188c175b6ec1ec" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaaae8f38bb311444cfb7f1979af0bc9240d95795f75f9ceddf6a59b79ceffa0" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.8", + "syn 1.0.55", +] + [[package]] name = "sct" version = "0.6.0" @@ -2722,9 +2979,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" dependencies = [ "block-buffer", "cfg-if 1.0.0", @@ -2742,7 +2999,7 @@ dependencies = [ "futures", "hashing", "lmdb", - "log 0.4.11", + "log 0.4.14", "task_executor", "tempfile", "tokio", @@ -2754,6 +3011,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb975283e6af8d3d691ddcefdca3a4dffe369167746d22fd993205e1e0c0de0" +[[package]] +name = "shellexpand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83bdb7831b2d85ddf4a7b148aa19d0587eddbe8671a436b7bd1182eaad0f2829" +dependencies = [ + "dirs-next", +] + [[package]] name = "shlex" version = "0.1.1" @@ -2816,6 +3082,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -2826,7 +3098,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" name = "stdio" version = "0.0.1" dependencies = [ - "log 0.4.11", + "log 0.4.14", "parking_lot", "tokio", ] @@ -2843,13 +3115,13 @@ dependencies = [ "criterion", "fs", "futures", - "glob", + "glob 0.2.11", "grpc_util", "hashing", "indexmap", "itertools 0.7.11", "lmdb", - "log 0.4.11", + "log 0.4.14", "maplit", "mock", "num_cpus", @@ -2981,6 +3253,16 @@ dependencies = [ "workunit_store", ] +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.2.0" @@ -3168,7 +3450,7 @@ dependencies = [ "bytes 1.0.1", "futures-core", "futures-sink", - "log 0.4.11", + "log 0.4.14", "pin-project-lite", "tokio", ] @@ -3264,7 +3546,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" dependencies = [ "cfg-if 1.0.0", - "log 0.4.11", + "log 0.4.14", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3465,7 +3747,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log 0.4.11", + "log 0.4.14", "try-lock", ] @@ -3501,7 +3783,7 @@ checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" dependencies = [ "bumpalo", "lazy_static", - "log 0.4.11", + "log 0.4.14", "proc-macro2 1.0.24", "quote 1.0.8", "syn 1.0.55", @@ -3549,6 +3831,12 @@ version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" +[[package]] +name = "wasmparser" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32fddd575d477c6e9702484139cf9f23dcd554b06d185ed0f56c857dd3a47aa6" + [[package]] name = "watch" version = "0.0.1" @@ -3557,7 +3845,7 @@ dependencies = [ "fs", "futures", "hashing", - "log 0.4.11", + "log 0.4.14", "notify", "parking_lot", "task_executor", @@ -3665,7 +3953,7 @@ dependencies = [ "concrete_time", "hashing", "hdrhistogram", - "log 0.4.11", + "log 0.4.14", "parking_lot", "petgraph 0.4.13", "rand 0.8.2", diff --git a/src/rust/engine/Cargo.toml b/src/rust/engine/Cargo.toml index 452f781d19b0..e332eff16ec1 100644 --- a/src/rust/engine/Cargo.toml +++ b/src/rust/engine/Cargo.toml @@ -22,6 +22,7 @@ members = [ "async_latch", "async_semaphore", "async_value", + "client", "concrete_time", "fs", "fs/brfs", @@ -59,6 +60,7 @@ default-members = [ ".", "async_latch", "async_semaphore", + "client", "async_value", "concrete_time", "fs", diff --git a/src/rust/engine/client/Cargo.toml b/src/rust/engine/client/Cargo.toml new file mode 100644 index 000000000000..a7806ac13914 --- /dev/null +++ b/src/rust/engine/client/Cargo.toml @@ -0,0 +1,31 @@ +[package] +version = "0.0.1" +edition = "2018" +name = "client" +authors = [ "Pants Build " ] +publish = false + +[[bin]] +name = "pants" +path = "src/main.rs" + +[dependencies] +env_logger = "0.5.4" +futures = "0.3" +libc = "0.2" +log = "0.4" +nails = "0.12" +nix = "0.20" +peg = "0.7" +remoteprocess = "0.4" +sha2 = "0.9" +shellexpand = "2.1" +strum = "0.20" +strum_macros = "0.20" +tokio = { version = "1.4", features = ["rt-multi-thread", "macros", "net", "io-std", "io-util"] } +toml = "0.5" +uname = "0.1" + +[dev-dependencies] +lazy_static = "1.4" +tempdir = "0.3" \ No newline at end of file diff --git a/src/rust/engine/client/src/build_root.rs b/src/rust/engine/client/src/build_root.rs new file mode 100644 index 000000000000..66f82caedc77 --- /dev/null +++ b/src/rust/engine/client/src/build_root.rs @@ -0,0 +1,165 @@ +// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +use log::debug; +use std::env; +use std::ops::Deref; +use std::path::PathBuf; + +#[derive(Debug)] +pub struct BuildRoot(PathBuf); + +impl BuildRoot { + const SENTINEL_FILES: [&'static str; 3] = ["pants", "BUILDROOT", "BUILD_ROOT"]; + + pub fn find() -> Result { + let cwd = env::current_dir().map_err(|e| format!("Failed to determine $CWD: {}", e))?; + let mut build_root = cwd.clone(); + loop { + for sentinel in &Self::SENTINEL_FILES { + let sentinel_path = build_root.join(sentinel); + if !sentinel_path.exists() { + continue; + } + let sentinel_path_metadata = sentinel_path.metadata().map_err(|e| { + format!( + "\ + Failed to read metadata for {path} to determine if is a build root sentinel file: {err}\ + ", + path = sentinel_path.display(), + err = e + ) + })?; + if sentinel_path_metadata.is_file() { + let root = BuildRoot(build_root); + debug!("Found {:?} starting search from {}.", root, cwd.display()); + return Ok(root); + } + } + + build_root = build_root + .parent() + .ok_or(format!( + "\ + No build root detected for the current directory of {cwd}. Pants detects the build root \ + by looking for at least one file from {sentinel_files} in the cwd and its ancestors. If \ + you have none of these files, you can create an empty file in your build root.\ + ", + cwd = cwd.display(), + sentinel_files = Self::SENTINEL_FILES.join(", ") + ))? + .into(); + } + } +} + +impl Deref for BuildRoot { + type Target = PathBuf; + + fn deref(&self) -> &PathBuf { + &self.0 + } +} + +#[cfg(test)] +mod test { + use crate::build_root::BuildRoot; + use lazy_static::lazy_static; + use std::ops::Deref; + use std::path::{Path, PathBuf}; + use std::sync::Mutex; + use std::{env, fs}; + use tempdir::TempDir; + + struct CurrentDir { + prior_cwd: Option, + cwd: PathBuf, + } + + impl CurrentDir { + fn cwd() -> CurrentDir { + CurrentDir { + prior_cwd: None, + cwd: env::current_dir().unwrap(), + } + } + + fn cd>(&mut self, path: P) -> CurrentDir { + let cwd = path.as_ref(); + fs::create_dir_all(cwd).unwrap(); + env::set_current_dir(cwd).unwrap(); + CurrentDir { + prior_cwd: Some(self.cwd.clone()), + cwd: cwd.into(), + } + } + + fn path(&self) -> &PathBuf { + &self.cwd + } + } + + impl Drop for CurrentDir { + fn drop(&mut self) { + if let Some(prior_cwd) = self.prior_cwd.take() { + env::set_current_dir(prior_cwd).unwrap(); + } + } + } + + impl Deref for CurrentDir { + type Target = PathBuf; + + fn deref(&self) -> &Self::Target { + &self.cwd + } + } + + lazy_static! { + // CWD is global to the process and Rust tests are multi-threaded by default; so we + // serialize tests here that need to mutate CWD. + static ref CWD: Mutex = Mutex::new(CurrentDir::cwd()); + } + + #[test] + fn test_find_cwd() { + let buildroot = TempDir::new("buildroot").unwrap(); + let mut cwd_lock = CWD.lock().unwrap(); + let cwd = cwd_lock.cd(buildroot.path()); + + let mut sentinel: Option = None; + + let mut assert_sentinel = |name| { + if let Some(prior_sentinel) = sentinel.take() { + fs::remove_file(prior_sentinel).unwrap(); + } + assert_eq!(buildroot.path(), env::current_dir().unwrap()); + assert!(BuildRoot::find().is_err()); + + let file = cwd.join(name); + fs::write(&file, &[]).unwrap(); + sentinel = Some(file); + assert_eq!(buildroot.path(), env::current_dir().unwrap()); + assert_eq!(buildroot.path(), BuildRoot::find().unwrap().as_path()); + }; + + assert_sentinel("BUILDROOT"); + assert_sentinel("BUILD_ROOT"); + assert_sentinel("pants"); + } + + #[test] + fn test_find_subdir() { + let buildroot = TempDir::new("buildroot").unwrap(); + let mut cwd_lock = CWD.lock().unwrap(); + let subdir = cwd_lock.cd(buildroot.path().join("foo").join("bar")); + + assert_eq!(subdir.path(), &env::current_dir().unwrap()); + assert!(BuildRoot::find().is_err()); + + let sentinel = &buildroot.path().join("pants"); + fs::write(&sentinel, &[]).unwrap(); + assert_eq!(subdir.path(), &env::current_dir().unwrap()); + assert_eq!(buildroot.path(), BuildRoot::find().unwrap().as_path()); + } +} diff --git a/src/rust/engine/client/src/client.rs b/src/rust/engine/client/src/client.rs new file mode 100644 index 000000000000..7152d3abeb0a --- /dev/null +++ b/src/rust/engine/client/src/client.rs @@ -0,0 +1,128 @@ +// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +use std::io; +use std::os::unix::io::AsRawFd; +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; + +use futures::channel::mpsc; +use futures::StreamExt; +use futures::{future, SinkExt, Stream, TryFutureExt}; +use log::debug; +use nails::execution::{child_channel, send_to_io, stream_for, ChildInput, ChildOutput, Command}; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpStream; + +pub struct ConnectionSettings { + pub port: u16, + pub timeout_limit: f64, + pub dynamic_ui: bool, +} + +impl ConnectionSettings { + pub fn default(port: u16) -> ConnectionSettings { + ConnectionSettings { + port, + timeout_limit: 60.0, + dynamic_ui: true, + } + } +} + +pub async fn execute_command( + start: SystemTime, + connection_settings: ConnectionSettings, + mut env: Vec<(String, String)>, + argv: Vec, + working_dir: &PathBuf, +) -> Result { + env.push(( + "PANTSD_RUNTRACKER_CLIENT_START_TIME".to_owned(), + start + .duration_since(SystemTime::UNIX_EPOCH) + .map_err(|e| format!("Failed to determine current unix time: {err}", err = e))? + .as_secs_f64() + .to_string(), + )); + env.push(( + "PANTSD_REQUEST_TIMEOUT_LIMIT".to_owned(), + connection_settings.timeout_limit.to_string(), + )); + if connection_settings.dynamic_ui { + for raw_fd in &[ + std::io::stdin().as_raw_fd(), + std::io::stdout().as_raw_fd(), + std::io::stderr().as_raw_fd(), + ] { + if let Ok(path) = nix::unistd::ttyname(*raw_fd) { + env.push(( + format!("NAILGUN_TTY_PATH_{fd}", fd = raw_fd), + path.display().to_string(), + )); + } + } + } + + let cmd = Command { + command: argv + .get(0) + .ok_or_else(|| "Failed to determine current process argv0".to_owned())? + .clone(), + args: argv.iter().skip(1).cloned().collect(), + env, + working_dir: working_dir.into(), + }; + + // TODO: This aligns with the C client. Possible that the default client and server configs + // should be different in order to be maximally lenient. + let config = nails::Config::default().heartbeat_frequency(Duration::from_millis(500)); + + debug!( + "Connecting to server at {address:?}...", + address = &connection_settings.port + ); + let stream = TcpStream::connect(("0.0.0.0", connection_settings.port)) + .await + .map_err(|e| format!("Error connecting to pantsd: {err}", err = e))?; + let mut child = nails::client::handle_connection(config, stream, cmd, async { + let (stdin_write, stdin_read) = child_channel::(); + let _join = tokio::spawn(handle_stdin(stdin_write)); + stdin_read + }) + .map_err(|e| format!("Error starting process: {err}", err = e)) + .await?; + + let output_stream = child.output_stream.take().unwrap(); + let stdio_printer = async move { tokio::spawn(handle_stdio(output_stream)).await.unwrap() }; + + future::try_join(stdio_printer, child.wait()) + .await + .map(|(_, exit_code)| exit_code.0) + .map_err(|e| format!("Error executing process: {err}", err = e)) +} + +async fn handle_stdio( + mut stdio_read: impl Stream + Unpin, +) -> Result<(), io::Error> { + let mut stdout = tokio::io::stdout(); + let mut stderr = tokio::io::stderr(); + while let Some(output) = stdio_read.next().await { + match output { + ChildOutput::Stdout(bytes) => stdout.write_all(&bytes).await?, + ChildOutput::Stderr(bytes) => stderr.write_all(&bytes).await?, + } + } + Ok(()) +} + +async fn handle_stdin(mut stdin_write: mpsc::Sender) -> Result<(), io::Error> { + let mut stdin = stream_for(tokio::io::stdin()); + while let Some(input_bytes) = stdin.next().await { + stdin_write + .send(ChildInput::Stdin(input_bytes?)) + .await + .map_err(send_to_io)?; + } + Ok(()) +} diff --git a/src/rust/engine/client/src/lib.rs b/src/rust/engine/client/src/lib.rs new file mode 100644 index 000000000000..57e269705824 --- /dev/null +++ b/src/rust/engine/client/src/lib.rs @@ -0,0 +1,44 @@ +// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +#![deny(warnings)] +// Enable all clippy lints except for many of the pedantic ones. It's a shame this needs to be copied and pasted across crates, but there doesn't appear to be a way to include inner attributes from a common source. +#![deny( + clippy::all, + clippy::default_trait_access, + clippy::expl_impl_clone_on_copy, + clippy::if_not_else, + clippy::needless_continue, + clippy::unseparated_literal_suffix, + // TODO: Falsely triggers for async/await: + // see https://github.com/rust-lang/rust-clippy/issues/5360 + // clippy::used_underscore_binding +)] +// It is often more clear to show that nothing is being moved. +#![allow(clippy::match_ref_pats)] +// Subjective style. +#![allow( + clippy::len_without_is_empty, + clippy::redundant_field_names, + clippy::too_many_arguments +)] +// Default isn't as big a deal as people seem to think it is. +#![allow(clippy::new_without_default, clippy::new_ret_no_self)] +// Arc can be more clear than needing to grok Orderings: +#![allow(clippy::mutex_atomic)] + +mod build_root; +mod client; +pub mod options; +pub mod pantsd; + +pub use crate::client::{execute_command, ConnectionSettings}; + +pub fn render_choice(items: &[&str]) -> Option { + match items { + [] => None, + [this] => Some(this.to_string()), + [this, that] => Some(format!("{} or {}", this, that)), + [these @ .., that] => Some(format!("{} or {}", these.join(", "), that)), + } +} diff --git a/src/rust/engine/client/src/main.rs b/src/rust/engine/client/src/main.rs new file mode 100644 index 000000000000..c733594e8095 --- /dev/null +++ b/src/rust/engine/client/src/main.rs @@ -0,0 +1,155 @@ +// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +#![deny(warnings)] +// Enable all clippy lints except for many of the pedantic ones. It's a shame this needs to be copied and pasted across crates, but there doesn't appear to be a way to include inner attributes from a common source. +#![deny( + clippy::all, + clippy::default_trait_access, + clippy::expl_impl_clone_on_copy, + clippy::if_not_else, + clippy::needless_continue, + clippy::unseparated_literal_suffix, + // TODO: Falsely triggers for async/await: + // see https://github.com/rust-lang/rust-clippy/issues/5360 + // clippy::used_underscore_binding +)] +// It is often more clear to show that nothing is being moved. +#![allow(clippy::match_ref_pats)] +// Subjective style. +#![allow( + clippy::len_without_is_empty, + clippy::redundant_field_names, + clippy::too_many_arguments +)] +// Default isn't as big a deal as people seem to think it is. +#![allow(clippy::new_without_default, clippy::new_ret_no_self)] +// Arc can be more clear than needing to grok Orderings: +#![allow(clippy::mutex_atomic)] + +use std::convert::AsRef; +use std::env; +use std::path::PathBuf; +use std::str::FromStr; +use std::time::SystemTime; + +use log::debug; +use strum::VariantNames; +use strum_macros::{AsRefStr, EnumString, EnumVariantNames}; + +use client::option_id; +use client::options::OptionParser; +use client::pantsd; +use client::render_choice; + +#[derive(AsRefStr, EnumString, EnumVariantNames)] +#[strum(serialize_all = "snake_case")] +enum PythonLogLevel { + TRACE = 5, + DEBUG = 10, + INFO = 20, + WARN = 30, + ERROR = 40, +} + +async fn execute(start: SystemTime) -> Result { + let options_parser = OptionParser::new()?; + + let use_pantsd = options_parser.parse_bool(&option_id!("pantsd"), true)?; + if !use_pantsd.value { + return Err(format!( + "Pantsd has been turned off via {option_source:?}.", + option_source = use_pantsd.source + )); + } + + let level_option = option_id!(-'l', "level"); + let log_level_option_value = + options_parser.parse_string(&level_option, PythonLogLevel::INFO.as_ref())?; + let level = PythonLogLevel::from_str(&log_level_option_value.value).map_err(|_| { + format!( + "Not a valid log level {level} from {option_source:?}. Should be one of {levels}.", + level = log_level_option_value.value, + option_source = log_level_option_value.source, + levels = render_choice(PythonLogLevel::VARIANTS) + .expect("We know there is at least one PythonLogLevel enum variant."), + ) + })?; + env_logger::init_from_env(env_logger::Env::new().filter_or("__PANTS_LEVEL__", level.as_ref())); + + let working_dir = env::current_dir() + .map_err(|e| format!("Could not detect current working directory: {err}", err = e))?; + let pantsd_settings = find_pantsd(&working_dir, &options_parser)?; + let env = env::vars().collect::>(); + let argv = env::args().collect::>(); + client::execute_command(start, pantsd_settings, env, argv, &working_dir).await +} + +fn find_pantsd( + working_dir: &PathBuf, + options_parser: &OptionParser, +) -> Result { + let pants_subprocessdir = option_id!("pants", "subprocessdir"); + let option_value = options_parser.parse_string(&pants_subprocessdir, ".pids")?; + let metadata_dir = { + let path = PathBuf::from(&option_value.value); + if path.is_absolute() { + path + } else { + match working_dir.join(&path) { + p if p.is_absolute() => p, + p => p.canonicalize().map_err(|e| { + format!( + "Failed to resolve relative pants subprocessdir specified via {:?} as {}: {}", + option_value, + path.display(), + e + ) + })?, + } + } + }; + debug!( + "\ + Looking for pantsd metadata in {metadata_dir} as specified by {option} = {value} via \ + {source:?}.\ + ", + metadata_dir = metadata_dir.display(), + option = pants_subprocessdir, + value = option_value.value, + source = option_value.source + ); + let port = pantsd::probe(&working_dir, &metadata_dir)?; + let mut pantsd_settings = client::ConnectionSettings::default(port); + pantsd_settings.timeout_limit = options_parser + .parse_float( + &option_id!("pantsd", "timeout", "when", "multiple", "invocations"), + pantsd_settings.timeout_limit, + )? + .value; + pantsd_settings.dynamic_ui = options_parser + .parse_bool(&option_id!("dynamic", "ui"), pantsd_settings.dynamic_ui)? + .value; + Ok(pantsd_settings) +} + +// The value is taken from this C precedent: +// ``` +// $ grep 75 /usr/include/sysexits.h +// #define EX_TEMPFAIL 75 /* temp failure; user is invited to retry */ +// ``` +const EX_TEMPFAIL: i32 = 75; + +#[tokio::main] +async fn main() { + let start = SystemTime::now(); + match execute(start).await { + Err(err) => { + eprintln!("{}", err); + // We use this exit code to indicate an error running pants via the nailgun protocol to + // differentiate from a successful nailgun protocol session. + std::process::exit(EX_TEMPFAIL); + } + Ok(exit_code) => std::process::exit(exit_code), + } +} diff --git a/src/rust/engine/client/src/options/args.rs b/src/rust/engine/client/src/options/args.rs new file mode 100644 index 000000000000..4daa95316233 --- /dev/null +++ b/src/rust/engine/client/src/options/args.rs @@ -0,0 +1,90 @@ +// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +use std::env; + +use super::id::{NameTransform, OptionId, Scope}; +use super::parse::parse_bool; +use super::OptionsSource; + +pub(crate) struct Args { + args: Vec, +} + +enum Negate { + TRUE, + FALSE, +} + +impl Args { + pub(crate) fn argv() -> Args { + Args { + args: env::args().collect::>(), + } + } + + fn arg_name(id: &OptionId, negate: Negate) -> String { + format!( + "--{}{}{}", + match negate { + Negate::FALSE => "", + Negate::TRUE => "no-", + }, + match &id.0 { + Scope::GLOBAL => "".to_string(), + Scope::Scope(scope) => format!("{}-", scope.to_ascii_lowercase()), + }, + id.name("-", NameTransform::ToLower) + ) + } + + fn find_string(&self, arg_name: &str) -> Result, String> { + for arg in &self.args { + let mut components = arg.as_str().splitn(2, '='); + if let Some(name) = components.next() { + if name == arg_name { + return Ok(Some(components.next().unwrap_or("").to_owned())); + } + } + } + Ok(None) + } +} + +impl OptionsSource for Args { + fn display(&self, id: &OptionId) -> String { + Self::arg_name(id, Negate::FALSE) + } + + fn get_string(&self, id: &OptionId) -> Result, String> { + if let Some(switch) = id.2 { + let prefixes = [format!("-{}=", switch), format!("-{}", switch)]; + for arg in &self.args { + for prefix in &prefixes { + if arg.starts_with(&*prefix) { + return Ok(Some(arg[prefix.len()..].to_owned())); + } + } + } + } + self.find_string(&Self::arg_name(id, Negate::FALSE)) + } + + fn get_bool(&self, id: &OptionId) -> Result, String> { + let arg_name = Self::arg_name(id, Negate::FALSE); + match self.find_string(&arg_name)? { + Some(s) if s.as_str() == "" => Ok(Some(true)), + Some(ref value) => parse_bool(&arg_name, value).map(Some), + None => { + let no_arg_name = Self::arg_name(id, Negate::TRUE); + match self.find_string(&no_arg_name)? { + Some(s) if s.as_str() == "" => Ok(Some(false)), + Some(ref value) => parse_bool(&no_arg_name, value) + .map(|value| !value) + .map(Some), + None => Ok(None), + } + } + } + } +} diff --git a/src/rust/engine/client/src/options/config.rs b/src/rust/engine/client/src/options/config.rs new file mode 100644 index 000000000000..ca32781fa6a8 --- /dev/null +++ b/src/rust/engine/client/src/options/config.rs @@ -0,0 +1,197 @@ +// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +use std::collections::HashSet; +use std::fs; +use std::path::Path; + +use toml::value::Table; +use toml::Value; + +use super::id::{NameTransform, OptionId}; +use super::{ListEdit, ListEditAction, OptionsSource}; + +#[derive(Clone)] +pub(crate) struct Config { + config: Value, +} + +impl Config { + pub(crate) fn default() -> Config { + Config { + config: Value::Table(Table::new()), + } + } + + pub(crate) fn parse>(file: P) -> Result { + let config_contents = fs::read_to_string(&file).map_err(|e| { + format!( + "Failed to read config file {}: {}", + file.as_ref().display(), + e + ) + })?; + let config = config_contents.parse::().map_err(|e| { + format!( + "Failed to parse config file {}: {}", + file.as_ref().display(), + e + ) + })?; + if config.is_table() { + Ok(Config { config }) + } else { + Err(format!( + "Expected the config file {} to contain a table but contained a {}: {}", + file.as_ref().display(), + config.type_str(), + config + )) + } + } + + pub(crate) fn merged>(files: &[P]) -> Result { + files + .iter() + .map(Config::parse) + .fold(Ok(Config::default()), |acc, parse_result| { + acc.and_then(|config| parse_result.map(|parsed| config.merge(parsed))) + }) + } + + fn option_name(id: &OptionId) -> String { + id.name("_", NameTransform::None) + } + + fn extract_string_list(option_name: &str, value: &Value) -> Result, String> { + if let Some(array) = value.as_array() { + let mut items = vec![]; + for item in array { + if let Some(value) = item.as_str() { + items.push(value.to_owned()) + } else { + return Err(format!( + "Expected {} to be an array of strings but given {} containing non string item {}", + option_name, value, item + )); + } + } + Ok(items) + } else { + Err(format!( + "Expected {} to be an array but given {}.", + option_name, value + )) + } + } + + fn get_value(&self, id: &OptionId) -> Option<&Value> { + self + .config + .get(&id.scope()) + .and_then(|table| table.get(Self::option_name(id))) + } + + pub(crate) fn merge(self, other: Config) -> Config { + let mut map = self.config.as_table().unwrap().to_owned(); + map.extend( + other + .config + .as_table() + .unwrap() + .iter() + .map(|(k, v)| (k.to_owned(), v.to_owned())), + ); + Config { + config: Value::Table(map), + } + } +} + +impl OptionsSource for Config { + fn get_string(&self, id: &OptionId) -> Result, String> { + if let Some(value) = self.get_value(id) { + if let Some(string) = value.as_str() { + Ok(Some(string.to_owned())) + } else { + Err(format!( + "Expected {} to be a string but given {}.", + id, value + )) + } + } else { + Ok(None) + } + } + + fn get_bool(&self, id: &OptionId) -> Result, String> { + if let Some(value) = self.get_value(id) { + if let Some(bool) = value.as_bool() { + Ok(Some(bool)) + } else { + Err(format!("Expected {} to be a bool but given {}.", id, value)) + } + } else { + Ok(None) + } + } + + fn get_float(&self, id: &OptionId) -> Result, String> { + if let Some(value) = self.get_value(id) { + if let Some(float) = value.as_float() { + Ok(Some(float)) + } else { + Err(format!( + "Expected {} to be a float but given {}.", + id, value + )) + } + } else { + Ok(None) + } + } + + fn get_string_list(&self, id: &OptionId) -> Result>>, String> { + if let Some(table) = self.config.get(&id.scope()) { + let option_name = Self::option_name(id); + let mut list_edits = vec![]; + if let Some(value) = table.get(&option_name) { + if let Some(sub_table) = value.as_table() { + if sub_table.is_empty() + || !sub_table.keys().collect::>().is_subset( + &["add".to_owned(), "remove".to_owned()] + .iter() + .collect::>(), + ) + { + return Err(format!( + "Expected {} to contain an 'add' element, a 'remove' element or both but found: {:?}", + option_name, sub_table + )); + } + if let Some(add) = sub_table.get("add") { + list_edits.push(ListEdit { + action: ListEditAction::ADD, + items: Self::extract_string_list(&*format!("{}.add", option_name), &add)?, + }) + } + if let Some(remove) = sub_table.get("remove") { + list_edits.push(ListEdit { + action: ListEditAction::REMOVE, + items: Self::extract_string_list(&*format!("{}.remove", option_name), &remove)?, + }) + } + } else { + list_edits.push(ListEdit { + action: ListEditAction::REPLACE, + items: Self::extract_string_list(&*option_name, value)?, + }); + } + } + if !list_edits.is_empty() { + return Ok(Some(list_edits)); + } + } + Ok(None) + } +} diff --git a/src/rust/engine/client/src/options/env.rs b/src/rust/engine/client/src/options/env.rs new file mode 100644 index 000000000000..3f6b1266fb36 --- /dev/null +++ b/src/rust/engine/client/src/options/env.rs @@ -0,0 +1,53 @@ +// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +use std::collections::HashMap; +use std::env; + +use super::id::{NameTransform, OptionId, Scope}; +use super::OptionsSource; + +#[derive(Debug)] +pub(crate) struct Env { + env: HashMap, +} + +impl Env { + pub(crate) fn capture() -> Env { + Env { + env: env::vars().collect::>(), + } + } + + fn env_var_names(id: &OptionId) -> Vec { + let name = id.name("_", NameTransform::ToUpper); + let mut names = vec![format!( + "PANTS_{}_{}", + id.0.name().to_ascii_uppercase(), + name + )]; + if id.0 == Scope::GLOBAL { + names.push(format!("PANTS_{}", name)); + } + if name.starts_with("PANTS_") { + names.push(name); + } + names + } +} + +impl OptionsSource for Env { + fn display(&self, id: &OptionId) -> String { + Self::env_var_names(id).pop().unwrap() + } + + fn get_string(&self, id: &OptionId) -> Result, String> { + let env_var_names = Self::env_var_names(id); + for env_var_name in &env_var_names { + if let Some(value) = self.env.get(env_var_name) { + return Ok(Some(value.to_owned())); + } + } + Ok(None) + } +} diff --git a/src/rust/engine/client/src/options/id.rs b/src/rust/engine/client/src/options/id.rs new file mode 100644 index 000000000000..293457e85a7d --- /dev/null +++ b/src/rust/engine/client/src/options/id.rs @@ -0,0 +1,180 @@ +// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +use std::fmt; +use std::fmt::{Display, Formatter}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Scope { + GLOBAL, + Scope(String), +} + +impl Scope { + pub fn named(name: &str) -> Scope { + match name { + "GLOBAL" => Scope::GLOBAL, + scope => Scope::Scope(scope.to_owned()), + } + } + + pub fn name(&self) -> &str { + match self { + Scope::GLOBAL => "GLOBAL", + Scope::Scope(scope) => scope.as_str(), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OptionId( + pub(crate) Scope, + pub(crate) Vec, + pub(crate) Option, +); + +impl OptionId { + pub fn new( + scope: Scope, + name: Name, + switch: Option, + ) -> Result + where + Component: AsRef, + Name: Iterator, + { + let name_components = name + .map(|component| component.as_ref().to_string()) + .collect::>(); + if name_components.is_empty() { + return Err(format!( + "Cannot create an OptionId with en empty name. Given a scope of {:?}.", + scope + )); + } + Ok(OptionId(scope, name_components, switch)) + } +} + +impl Display for OptionId { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "[{}] {}", + self.scope(), + self.name("_", NameTransform::None) + ) + } +} + +#[macro_export] +macro_rules! option_id { + (-$switch:literal, [$scope:literal], $($name_component:literal),+) => { + $crate::options::OptionId::new( + $crate::options::Scope::named($scope), + [$($name_component),+].iter(), + Some($switch) + ).expect("Creating an OptionId via macro should ensure at least one name component") + }; + (-$switch:literal, $($name_component:literal),+) => { + $crate::options::OptionId::new( + $crate::options::Scope::GLOBAL, + [$($name_component),+].iter(), + Some($switch) + ).expect("Creating an OptionId via macro should ensure at least one name component") + }; + ([$scope:literal], $($name_component:literal),+) => { + $crate::options::OptionId::new( + $crate::options::Scope::named($scope), + [$($name_component),+].iter(), + None + ).expect("Creating an OptionId via macro should ensure at least one name component") + }; + ($($name_component:literal),+) => { + $crate::options::OptionId::new( + $crate::options::Scope::GLOBAL, + [$($name_component),+].iter(), + None + ).expect("Creating an OptionId via macro should ensure at least one name component") + }; +} + +pub(crate) enum NameTransform { + None, + ToLower, + ToUpper, +} + +impl OptionId { + pub(crate) fn scope(&self) -> &str { + self.0.name() + } + + pub(crate) fn name(&self, sep: &str, transform: NameTransform) -> String { + self + .1 + .iter() + .map(|component| match transform { + NameTransform::None => component.to_owned(), + NameTransform::ToLower => component.to_ascii_lowercase(), + NameTransform::ToUpper => component.to_ascii_uppercase(), + }) + .collect::>() + .join(sep) + } +} + +#[cfg(test)] +mod test { + use crate::options::id::{OptionId, Scope}; + + #[test] + fn test_option_id_global_switch() { + let option_id = option_id!(-'x', "bar", "baz"); + assert_eq!( + OptionId::new(Scope::GLOBAL, ["bar", "baz"].iter(), Some('x')).unwrap(), + option_id + ); + assert_eq!("GLOBAL", option_id.scope()); + } + + #[test] + fn test_option_id_global() { + let option_id = option_id!("bar", "baz"); + assert_eq!( + OptionId::new(Scope::GLOBAL, ["bar", "baz"].iter(), None).unwrap(), + option_id + ); + assert_eq!("GLOBAL", option_id.scope()); + } + + #[test] + fn test_option_id_scope_switch() { + let option_id = option_id!(-'f', ["foo-bar"], "baz", "spam"); + assert_eq!( + OptionId::new( + Scope::Scope("foo-bar".to_owned()), + ["baz", "spam"].iter(), + Some('f') + ) + .unwrap(), + option_id + ); + assert_eq!("foo-bar", option_id.scope()); + } + + #[test] + fn test_option_id_scope() { + let option_id = option_id!(["foo-bar"], "baz", "spam"); + assert_eq!( + OptionId::new( + Scope::Scope("foo-bar".to_owned()), + ["baz", "spam"].iter(), + None + ) + .unwrap(), + option_id + ); + assert_eq!("foo-bar", option_id.scope()); + } +} diff --git a/src/rust/engine/client/src/options/mod.rs b/src/rust/engine/client/src/options/mod.rs new file mode 100644 index 000000000000..4b332bda421f --- /dev/null +++ b/src/rust/engine/client/src/options/mod.rs @@ -0,0 +1,223 @@ +// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +mod args; +mod config; +mod env; +mod id; +mod parse; + +use std::collections::{BTreeMap, HashSet}; +use std::ops::Deref; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; +use std::rc::Rc; + +use crate::build_root::BuildRoot; +use crate::option_id; + +use args::Args; +use config::Config; +use env::Env; +use parse::{parse_bool, parse_string_list}; + +pub use id::{OptionId, Scope}; + +#[derive(Copy, Clone, Debug)] +pub(crate) enum ListEditAction { + REPLACE, + ADD, + REMOVE, +} + +#[derive(Debug)] +pub(crate) struct ListEdit { + pub action: ListEditAction, + pub items: Vec, +} + +trait OptionsSource { + fn display(&self, id: &OptionId) -> String { + format!("{}", id) + } + + fn get_string(&self, id: &OptionId) -> Result, String>; + + fn get_bool(&self, id: &OptionId) -> Result, String> { + if let Some(value) = self.get_string(id)? { + parse_bool(&*self.display(id), &*value).map(Some) + } else { + Ok(None) + } + } + + fn get_float(&self, id: &OptionId) -> Result, String> { + if let Some(value) = self.get_string(id)? { + value.parse().map(Some).map_err(|e| { + format!( + "Problem parsing {} value {} as a float value: {}", + self.display(id), + value, + e + ) + }) + } else { + Ok(None) + } + } + + fn get_string_list(&self, id: &OptionId) -> Result>>, String> { + if let Some(value) = self.get_string(id)? { + parse_string_list(&*self.display(id), &value).map(Some) + } else { + Ok(None) + } + } +} + +#[derive(Copy, Clone, Debug, Ord, PartialOrd, Eq, PartialEq)] +pub enum Source { + FLAG, + ENV, + CONFIG, + DEFAULT, +} + +#[derive(Debug)] +pub struct OptionValue { + pub source: Source, + pub value: T, +} + +impl Deref for OptionValue { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.value + } +} + +pub struct OptionParser { + sources: BTreeMap>, +} + +impl OptionParser { + pub fn new() -> Result { + let mut sources: BTreeMap> = BTreeMap::new(); + sources.insert(Source::ENV, Rc::new(Env::capture())); + sources.insert(Source::FLAG, Rc::new(Args::argv())); + let mut parser = OptionParser { + sources: sources.clone(), + }; + + let config_path = BuildRoot::find()?.join("pants.toml"); + let repo_config_files = parser.parse_string_list( + &option_id!("pants", "config", "files"), + &[ + std::str::from_utf8(config_path.as_os_str().as_bytes()).map_err(|e| { + format!( + "Failed to decode build root path {}: {}", + config_path.display(), + e + ) + })?, + ], + )?; + let mut config = Config::merged(&repo_config_files)?; + sources.insert(Source::CONFIG, Rc::new(config.clone())); + parser = OptionParser { + sources: sources.clone(), + }; + + if *parser.parse_bool(&option_id!("pantsrc"), true)? { + for rcfile in parser.parse_string_list( + &option_id!("pantsrc", "files"), + &["/etc/pantsrc", shellexpand::tilde("~/.pants.rc").as_ref()], + )? { + let rcfile_path = Path::new(&rcfile); + if rcfile_path.exists() { + let rc_config = Config::parse(rcfile_path)?; + config = config.merge(rc_config); + } + } + } + sources.insert(Source::CONFIG, Rc::new(config)); + Ok(OptionParser { sources }) + } + + pub fn parse_bool(&self, id: &OptionId, default: bool) -> Result, String> { + for (source_type, source) in self.sources.iter() { + if let Some(value) = source.get_bool(id)? { + return Ok(OptionValue { + source: *source_type, + value, + }); + } + } + Ok(OptionValue { + source: Source::DEFAULT, + value: default, + }) + } + + pub fn parse_float(&self, id: &OptionId, default: f64) -> Result, String> { + for (source_type, source) in self.sources.iter() { + if let Some(value) = source.get_float(id)? { + return Ok(OptionValue { + source: *source_type, + value, + }); + } + } + Ok(OptionValue { + source: Source::DEFAULT, + value: default, + }) + } + + pub fn parse_string(&self, id: &OptionId, default: &str) -> Result, String> { + for (source_type, source) in self.sources.iter() { + if let Some(value) = source.get_string(id)? { + return Ok(OptionValue { + source: *source_type, + value, + }); + } + } + Ok(OptionValue { + source: Source::DEFAULT, + value: default.to_string(), + }) + } + + pub fn parse_string_list(&self, id: &OptionId, default: &[&str]) -> Result, String> { + let mut list_edits = vec![]; + for (_, source) in self.sources.iter() { + if let Some(edits) = source.get_string_list(id)? { + for edit in edits { + list_edits.push(edit); + } + } + } + let mut string_list = default.iter().map(|s| s.to_string()).collect::>(); + for list_edit in list_edits { + match list_edit.action { + ListEditAction::REPLACE => string_list = list_edit.items, + ListEditAction::ADD => { + for item in list_edit.items { + string_list.push(item); + } + } + ListEditAction::REMOVE => { + let to_remove = list_edit.items.iter().collect::>(); + string_list = string_list + .iter() + .filter(|item| !to_remove.contains(item)) + .map(|s| s.to_owned()) + .collect::>(); + } + } + } + Ok(string_list) + } +} diff --git a/src/rust/engine/client/src/options/parse.rs b/src/rust/engine/client/src/options/parse.rs new file mode 100644 index 000000000000..b762efed3cd8 --- /dev/null +++ b/src/rust/engine/client/src/options/parse.rs @@ -0,0 +1,152 @@ +// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +use super::{ListEdit, ListEditAction}; +use crate::render_choice; + +peg::parser! { + grammar option_value_parser() for str { + use peg::ParseLiteral; + + rule whitespace() -> () + = quiet!{ " " / "\n" / "\r" / "\t" } + + rule string() -> String + = whitespace()* + string:(double_quoted_string() / single_quoted_string()) + whitespace()* { string } + + rule double_quoted_string() -> String + = "\"" s:double_quoted_character()* "\"" { s.into_iter().collect() } + + rule double_quoted_character() -> char + = quoted_character("\"") + / escaped_character() + + rule single_quoted_string() -> String + = "'" s:single_quoted_character()* "'" { s.into_iter().collect() } + + rule single_quoted_character() -> char + = quoted_character("'") + / escaped_character() + + rule quoted_character(quote_char: &'static str) -> char + = !(##parse_string_literal(quote_char) / "\\") c:$([_]) { c.chars().next().unwrap() } + + rule escaped_character() -> char + = "\\" c:$([_]) { c.chars().next().unwrap() } + + rule add() -> ListEditAction + = "+" { ListEditAction::ADD } + + rule remove() -> ListEditAction + = "-" { ListEditAction::REMOVE } + + rule action() -> ListEditAction + = quiet!{ action:(add() / remove()) { action } } + / expected!( + "an optional list edit action of '+' indicating `add` or '-' indicating `remove`" + ) + + // N.B.: The Python list parsing implementation accepts Python tuple literal syntax too. + + rule tuple_start() -> () + = quiet!{ "(" } + / expected!("the start of a tuple indicated by '('") + + rule tuple_end() -> () + = quiet!{ ")" } + / expected!("the end of a tuple indicated by ')'") + + rule tuple_items() -> Vec + = tuple_start() items:string() ** "," ","? tuple_end() { items } + + rule list_start() -> () + = quiet!{ "[" } + / expected!("the start of a list indicated by '['") + + rule list_end() -> () + = quiet!{ "]" } + / expected!("the end of a list indicated by ']'") + + rule list_items() -> Vec + = list_start() items:string() ** "," ","? list_end() { items } + + rule items() -> Vec + = tuple_items() + / list_items() + + rule list_edit() -> ListEdit + = whitespace()* action:action() items:items() whitespace()* { + ListEdit { action, items } + } + + rule list_edits() -> Vec> + = e:list_edit() ** "," ","? { e } + + rule list_replace() -> Vec> + = items:items() { + vec![ListEdit { action: ListEditAction::REPLACE, items }] + } + + rule implicit_add() -> Vec> + = !(whitespace() / add() / remove() / tuple_start() / list_start()) item:$([_]+) { + vec![ListEdit { action: ListEditAction::ADD, items: vec![item.to_owned()] }] + } + + pub(crate) rule string_list_edits() -> Vec> + = list_edits() / list_replace() / implicit_add() + } +} + +fn format_parse_error( + id: &str, + type_id: &str, + value: &str, + parse_error: peg::error::ParseError, +) -> String { + let value_with_marker = value + .split('\n') + .enumerate() + .map(|(index, line)| (index + 1, line)) + .map(|(line_no, line)| { + if line_no == parse_error.location.line { + format!( + "{}:{}\n {}^", + line_no, + line, + "-".repeat(parse_error.location.column - 1) + ) + } else { + format!("{}:{}", line_no, line) + } + }) + .collect::>() + .join("\n"); + format!( + "Problem parsing {} {} value:\n{}\nExpected {} at line {} column {}", + id, + type_id, + value_with_marker, + render_choice(parse_error.expected.tokens().collect::>().as_slice()) + .unwrap_or_else(|| "nothing".to_owned()), + parse_error.location.line, + parse_error.location.column, + ) +} + +pub(crate) fn parse_string_list(name: &str, value: &str) -> Result>, String> { + option_value_parser::string_list_edits(&value) + .map_err(|e| format_parse_error(name, "string list", &*value, e)) +} + +pub(crate) fn parse_bool(name: &str, value: &str) -> Result { + match value.to_lowercase().as_str() { + "true" => Ok(true), + "false" => Ok(false), + _ => Err(format!( + "Got '{}' for {}. Expected 'true' or 'false'.", + value, name + )), + } +} diff --git a/src/rust/engine/client/src/pantsd.rs b/src/rust/engine/client/src/pantsd.rs new file mode 100644 index 000000000000..617ec1100a8b --- /dev/null +++ b/src/rust/engine/client/src/pantsd.rs @@ -0,0 +1,247 @@ +// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +use core::fmt; +use std::fs; +use std::path::{Path, PathBuf}; + +use libc::pid_t; +use log::debug; +use sha2::{Digest, Sha256}; + +struct Metadata { + metadata_dir: PathBuf, +} + +impl Metadata { + fn mount>(directory: P) -> Result { + let info = uname::uname().map_err(|e| format!("{}", e))?; + let host_hash = Sha256::new() + .chain(&info.sysname) + .chain(&info.nodename) + .chain(&info.release) + .chain(&info.version) + .chain(&info.machine) + .finalize(); + + const HOST_FINGERPRINT_LENGTH: usize = 12; + let mut hex_digest = String::with_capacity(HOST_FINGERPRINT_LENGTH); + for byte in host_hash { + fmt::Write::write_fmt(&mut hex_digest, format_args!("{:02x}", byte)).unwrap(); + if hex_digest.len() >= HOST_FINGERPRINT_LENGTH { + break; + } + } + + let metadata_dir = directory + .as_ref() + .join(&hex_digest[..HOST_FINGERPRINT_LENGTH]) + .join("pantsd"); + if metadata_dir.is_dir() { + Ok(Metadata { metadata_dir }) + } else { + Err(format!( + "There is no pantsd metadata at {metadata_dir}.", + metadata_dir = metadata_dir.display() + )) + } + } + + fn pid(&self) -> Result { + self + .read_metadata("pid") + .and_then(|(pid_metadata_path, value)| { + value + .parse() + .map(|pid| { + debug!( + "Parsed pid {pid} from {pid_metadata_path}.", + pid = pid, + pid_metadata_path = pid_metadata_path.display() + ); + pid + }) + .map_err(|e| { + format!( + "Failed to parse pantsd pid from {pid_metadata_path}: {err}", + pid_metadata_path = pid_metadata_path.display(), + err = e + ) + }) + }) + } + + fn process_name(&self) -> Result { + self.read_metadata("process_name").map(|(_, value)| value) + } + + fn port(&self) -> Result { + self + .read_metadata("socket") + .and_then(|(socket_metadata_path, value)| { + value + .parse() + .map(|port| { + debug!( + "Parsed port {port} from {socket_metadata_path}.", + port = port, + socket_metadata_path = socket_metadata_path.display() + ); + port + }) + .map_err(|e| { + format!( + "Failed to parse pantsd port from {socket_metadata_path}: {err}", + socket_metadata_path = &socket_metadata_path.display(), + err = e + ) + }) + }) + } + + fn read_metadata(&self, name: &str) -> Result<(PathBuf, String), String> { + let metadata_path = self.metadata_dir.join(name); + fs::read_to_string(&metadata_path) + .map_err(|e| { + format!( + "Failed to read {name} from {metadata_path}: {err}", + name = name, + metadata_path = &metadata_path.display(), + err = e + ) + }) + .map(|value| (metadata_path, value)) + } +} + +pub fn probe(working_dir: &Path, metadata_dir: &Path) -> Result { + let pantsd_metadata = Metadata::mount(metadata_dir)?; + + // Grab the purported port early. If we can't get that, then none of the following checks + // are useful. + let port = pantsd_metadata.port()?; + + // Check that the recorded pid is a live process. + // TODO(John Sirois): This does not check for zombie status like the psutil-based Python + // implementation does. + let pid = pantsd_metadata.pid()?; + nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid), None).map_err(|e| { + format!( + "\ + The last pid for the pantsd controlling {working_dir} was {pid} but it's no longer running: \ + {err}\ + ", + working_dir = working_dir.display(), + pid = pid, + err = e + ) + })?; + + // Check that the live process is in fact the expected pantsd process (i.e.: pids have not + // wrapped). + let pantsd_process = remoteprocess::Process::new(pid).map_err(|e| { + format!( + "Failed to read process information for pantsd at pid {pid}: {err}", + pid = pid, + err = e + ) + })?; + let expected_process_name_prefix = pantsd_metadata.process_name()?; + let actual_command_line = pantsd_process.cmdline().map_err(|e| { + format!( + "Failed to determine the process name for the running process at pid {pid}: {err}", + pid = pid, + err = e + ) + })?; + let actual_argv0 = actual_command_line.get(0).ok_or_else(|| { + format!( + "The command line for pantsd at pid {pid} was unexpectedly empty.", + pid = pid + ) + })?; + // It appears the the daemon only records a prefix of the process name, so we just check that. + if actual_argv0.starts_with(&expected_process_name_prefix) { + Ok(port) + } else { + Err(format!( + "\ + The process with pid {pid} is not pantsd. Expected a process name matching {expected_name} \ + but is {actual_name}.\ + ", + pid = pid, + expected_name = expected_process_name_prefix, + actual_name = actual_argv0 + )) + } +} + +#[cfg(test)] +mod test { + use crate::build_root::BuildRoot; + use crate::pantsd; + use std::fs; + use std::net::TcpStream; + use std::process::{Command, Stdio}; + use std::str::from_utf8; + use tempdir::TempDir; + + fn launch_pantsd() -> (BuildRoot, TempDir) { + let build_root = BuildRoot::find() + .expect("Expected test to be run inside the Pants repo but no build root was detected."); + let pants_subprocessdir = TempDir::new("pants_subproccessdir").unwrap(); + let mut cmd = Command::new(build_root.join("pants")); + cmd + .current_dir(build_root.as_path()) + .arg("--pants-config-files=[]") + .arg("--no-pantsrc") + .arg("--pantsd") + .arg(format!( + "--pants-subprocessdir={}", + pants_subprocessdir.path().display() + )) + .arg("-V") + .stderr(Stdio::inherit()); + let result = cmd.output().unwrap(); + assert_eq!(Some(0), result.status.code()); + assert_eq!( + fs::read_to_string( + build_root + .join("src") + .join("python") + .join("pants") + .join("VERSION") + ) + .unwrap(), + from_utf8(result.stdout.as_slice()).unwrap() + ); + (build_root, pants_subprocessdir) + } + + fn assert_connect(port: u16) { + assert!( + port >= 1024, + "Pantsd should never be running on a privileged port." + ); + + let stream = TcpStream::connect(("0.0.0.0", port)).unwrap(); + assert_eq!(port, stream.peer_addr().unwrap().port()); + } + + #[test] + fn test_address_integration() { + let (_, pants_subprocessdir) = launch_pantsd(); + + let pantsd_metadata = pantsd::Metadata::mount(&pants_subprocessdir).unwrap(); + let port = pantsd_metadata.port().unwrap(); + assert_connect(port); + } + + #[test] + fn test_probe() { + let (build_root, pants_subprocessdir) = launch_pantsd(); + + let port = pantsd::probe(&build_root, pants_subprocessdir.path()).unwrap(); + assert_connect(port); + } +}