diff --git a/.gitignore b/.gitignore index 3d4a8febfbe381..69790d143ea8e6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,9 @@ Cargo.lock /target/ **/*.rs.bk .cargo + +# node configuration files +/config/ +/config-drone/ +/config-validator/ +/config-client-demo/ diff --git a/README.md b/README.md index 5555776165183e..9b43ea14182fae 100644 --- a/README.md +++ b/README.md @@ -84,13 +84,12 @@ Now start the server: $ ./multinode-demo/leader.sh ``` -To run a performance-enhanced fullnode on Linux, download `libcuda_verify_ed25519.a`. Enable -it by adding `--features=cuda` to the line that runs `solana-fullnode` in -`leader.sh`. [CUDA 9.2](https://developer.nvidia.com/cuda-downloads) must be installed on your system. - +To run a performance-enhanced fullnode on Linux, +[CUDA 9.2](https://developer.nvidia.com/cuda-downloads) must be installed on +your system: ```bash $ ./fetch-perf-libs.sh -$ cargo run --release --features=cuda --bin solana-fullnode -- -l leader.json < genesis.log +$ SOLANA_CUDA=1 ./multinode-demo/leader.sh ``` Wait a few seconds for the server to initialize. It will print "Ready." when it's ready to @@ -102,14 +101,15 @@ Multinode Testnet To run a multinode testnet, after starting a leader node, spin up some validator nodes: ```bash -$ ./multinode-demo/validator.sh ubuntu@10.0.1.51:~/solana #The leader machine +$ ./multinode-demo/validator.sh ubuntu@10.0.1.51:~/solana 10.0.1.51 ``` -As with the leader node, you can run a performance-enhanced validator fullnode by adding -`--features=cuda` to the line that runs `solana-fullnode` in `validator.sh`. - +To run a performance-enhanced fullnode on Linux, +[CUDA 9.2](https://developer.nvidia.com/cuda-downloads) must be installed on +your system: ```bash -$ cargo run --release --features=cuda --bin solana-fullnode -- -l validator.json -v leader.json < genesis.log +$ ./fetch-perf-libs.sh +$ SOLANA_CUDA=1 ./multinode-demo/leader.sh ubuntu@10.0.1.51:~/solana 10.0.1.51 ``` @@ -152,6 +152,70 @@ $ snap info solana $ sudo snap refresh solana --devmode ``` +### Daemon support +The snap supports running a leader, validator or leader+drone node as a system +daemon. + +Run `sudo snap get solana` to view the current daemon configuration, and +`sudo snap logs -f solana` to view the daemon logs. + +Disable the daemon at any time by running: +```bash +$ sudo snap set solana mode= +``` + +Runtime configuration files for the daemon can be found in +`/var/snap/solana/current/config`. + +#### Leader daemon +```bash +$ sudo snap set solana mode=leader +``` + +If CUDA is available: +```bash +$ sudo snap set solana mode=leader enable-cuda=1 +``` + +`rsync` must be configured and running on the leader. + +1. Ensure rsync is installed with `sudo apt-get -y install rsync` +2. Edit `/etc/rsyncd.conf` to include the following +``` +[config] +path = /var/snap/solana/current/config +hosts allow = * +read only = true +``` +3. Run `sudo systemctl enable rsync; sudo systemctl start rsync` +4. Test by running `rsync -Pzravv rsync:///config +solana-config` from another machine. If the leader is running on a cloud +provider it may be necessary to configure the Firewall rules to permit ingress +to port tcp:873, tcp:9900 and the port range udp:8000-udp:10000 + + +To run both the Leader and Drone: +```bash +$ sudo snap set solana mode=leader+drone +``` + +#### Validator daemon +```bash +$ sudo snap set solana mode=validator +``` +If CUDA is available: +```bash +$ sudo snap set solana mode=validator enable-cuda=1 +``` + +By default the validator will connect to **testnet.solana.com**, override +the leader IP address by running: +```bash +$ sudo snap set solana mode=validator leader-address=127.0.0.1 #<-- change IP address +``` +It's assumed that the leader will be running `rsync` configured as described in +the previous **Leader daemon** section. + Developing === diff --git a/ci/snap.sh b/ci/snap.sh index c12a9f2801dff0..8d68c77ec93574 100755 --- a/ci/snap.sh +++ b/ci/snap.sh @@ -36,5 +36,8 @@ set -x echo --- build snapcraft +source ci/upload_ci_artifact.sh +upload_ci_artifact solana_*.snap + echo --- publish $DRYRUN snapcraft push solana_*.snap --release $SNAP_CHANNEL diff --git a/ci/upload_ci_artifact.sh b/ci/upload_ci_artifact.sh new file mode 100644 index 00000000000000..9bbdce3c7369b0 --- /dev/null +++ b/ci/upload_ci_artifact.sh @@ -0,0 +1,18 @@ +# |source| me + +upload_ci_artifact() { + echo "--- artifact: $1" + if [[ -r "$1" ]]; then + ls -l "$1" + if ${BUILDKITE:-false}; then + ( + set -x + buildkite-agent artifact upload "$1" + ) + fi + else + echo ^^^ +++ + echo "$1 not found" + fi +} + diff --git a/multinode-demo/client.sh b/multinode-demo/client.sh index ef69e6de4eeb6d..b358fe8892a1cf 100755 --- a/multinode-demo/client.sh +++ b/multinode-demo/client.sh @@ -1,17 +1,22 @@ #!/bin/bash +# +# usage: $0 " +# -if [[ -z $1 ]]; then - echo "usage: $0 [network path to solana repo on leader machine] " - exit 1 -fi +here=$(dirname "$0") +# shellcheck source=multinode-demo/common.sh +source "$here"/common.sh +SOLANA_CONFIG_DIR=config-client-demo -LEADER=$1 -COUNT=${2:-1} +leader=${1:-${here}/..} # Default to local solana repo +count=${2:-1} -rsync -vz "$LEADER"/{leader.json,mint-demo.json} . || exit $? +set -ex +mkdir -p $SOLANA_CONFIG_DIR +rsync -vz "$leader"/config/leader.json $SOLANA_CONFIG_DIR/ +rsync -vz "$leader"/config/mint-demo.json $SOLANA_CONFIG_DIR/ -# if RUST_LOG is unset, default to info -export RUST_LOG=${RUST_LOG:-solana=info} - -cargo run --release --bin solana-client-demo -- \ - -n "$COUNT" -l leader.json -d < mint-demo.json 2>&1 | tee client.log +# shellcheck disable=SC2086 # $solana_client_demo should not be quoted +exec $solana_client_demo \ + -n "$count" -l $SOLANA_CONFIG_DIR/leader.json -d \ + < $SOLANA_CONFIG_DIR/mint-demo.json 2>&1 | tee client-demo.log diff --git a/multinode-demo/common.sh b/multinode-demo/common.sh new file mode 100644 index 00000000000000..c79235ed525139 --- /dev/null +++ b/multinode-demo/common.sh @@ -0,0 +1,36 @@ +# |source| this file +# +# Disable complaints about unused variables in this file: +# shellcheck disable=2034 + +if [[ -d "$SNAP" ]]; then # Running as a Linux Snap? + solana_program() { + local program="$1" + printf "%s/command-%s.wrapper" "$SNAP" "$program" + } + SOLANA_CUDA="$(snapctl get enable-cuda)" +else + solana_program() { + local program="$1" + local features="" + if [[ "$program" =~ ^(.*)-cuda$ ]]; then + program=${BASH_REMATCH[1]} + features="--features=cuda,erasure" + fi + printf "cargo run --release --bin solana-%s %s -- " "$program" "$features" + } +fi + +solana_client_demo=$(solana_program client-demo) +solana_drone=$(solana_program drone) +solana_fullnode=$(solana_program fullnode) +solana_fullnode_config=$(solana_program fullnode-config) +solana_fullnode_cuda=$(solana_program fullnode-cuda) +solana_genesis_demo=$(solana_program genesis-demo) +solana_mint_demo=$(solana_program mint-demo) + +export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info +export RUST_BACKTRACE=1 +[[ $(uname) = Linux ]] && (set -x; sudo sysctl -w net.core.rmem_max=26214400 1>/dev/null 2>/dev/null) + +SOLANA_CONFIG_DIR=${SNAP_DATA:-$PWD}/config diff --git a/multinode-demo/drone.sh b/multinode-demo/drone.sh new file mode 100755 index 00000000000000..159fed5e79a374 --- /dev/null +++ b/multinode-demo/drone.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# usage: $0 +# + +here=$(dirname "$0") +# shellcheck source=multinode-demo/common.sh +source "$here"/common.sh +SOLANA_CONFIG_DIR=config-drone + +if [[ -d "$SNAP" ]]; then + # Select leader from the Snap configuration + leader_address="$(snapctl get leader-address)" + if [[ -z "$leader_address" ]]; then + # Assume drone is running on the same node as the leader by default + leader_address="localhost" + fi + leader=rsync://"$leader_address" +else + leader=${1:-${here}/..} # Default to local solana repo +fi + +set -ex +mkdir -p $SOLANA_CONFIG_DIR +rsync -vz "$leader"/config/leader.json $SOLANA_CONFIG_DIR/ +rsync -vz "$leader"/config/mint-demo.json $SOLANA_CONFIG_DIR/ + +# shellcheck disable=SC2086 # $solana_drone should not be quoted +exec $solana_drone \ + -l $SOLANA_CONFIG_DIR/leader.json < $SOLANA_CONFIG_DIR/mint-demo.json diff --git a/multinode-demo/leader.sh b/multinode-demo/leader.sh index 6d9e5a098e9f75..9587899bb5af5f 100755 --- a/multinode-demo/leader.sh +++ b/multinode-demo/leader.sh @@ -1,28 +1,22 @@ #!/bin/bash -here=$(dirname "$0") - -# shellcheck source=/dev/null -. "${here}"/myip.sh -myip=$(myip) || exit $? +here=$(dirname "$0") +# shellcheck source=multinode-demo/common.sh +source "$here"/common.sh -[[ -f leader-"${myip}".json ]] || { - echo "I can't find a matching leader config file for \"${myip}\"... -Please run ${here}/setup.sh first. -" +[[ -f "$SOLANA_CONFIG_DIR"/leader.json ]] || { + echo "$SOLANA_CONFIG_DIR/leader.json not found, run ${here}/setup.sh first" exit 1 } -# if RUST_LOG is unset, default to info -export RUST_LOG=${RUST_LOG:-solana=info} - -[[ $(uname) = Linux ]] && sudo sysctl -w net.core.rmem_max=26214400 1>/dev/null 2>/dev/null - -# this makes a leader.json file available alongside genesis, etc. for -# validators and clients -cp leader-"${myip}".json leader.json +if [[ -n "$SOLANA_CUDA" ]]; then + program="$solana_fullnode_cuda" +else + program="$solana_fullnode" +fi -cargo run --release --bin solana-fullnode -- \ - -l leader-"${myip}".json \ - < genesis.log tx-*.log \ - > tx-"$(date -u +%Y%m%d%H%M%S%N)".log +# shellcheck disable=SC2086 # $program should not be quoted +exec $program \ + -l "$SOLANA_CONFIG_DIR"/leader.json \ + < "$SOLANA_CONFIG_DIR"/genesis.log "$SOLANA_CONFIG_DIR"/tx-*.log \ + > "$SOLANA_CONFIG_DIR"/tx-"$(date -u +%Y%m%d%H%M%S%N)".log diff --git a/multinode-demo/setup.sh b/multinode-demo/setup.sh index 3c68abccab1159..c6c347e96b4869 100755 --- a/multinode-demo/setup.sh +++ b/multinode-demo/setup.sh @@ -1,15 +1,27 @@ #!/bin/bash + +num_tokens=${1:-1000000000} + here=$(dirname "$0") +# shellcheck source=multinode-demo/common.sh +source "$here"/common.sh -# shellcheck source=/dev/null -. "${here}"/myip.sh +set -e -myip=$(myip) || exit $? +echo "Cleaning $SOLANA_CONFIG_DIR" +rm -rvf "$SOLANA_CONFIG_DIR" +mkdir -p "$SOLANA_CONFIG_DIR" -num_tokens=${1:-1000000000} +echo "Creating $SOLANA_CONFIG_DIR/mint-demo.json with $num_tokens tokens" +$solana_mint_demo <<<"$num_tokens" > "$SOLANA_CONFIG_DIR"/mint-demo.json + +echo "Creating $SOLANA_CONFIG_DIR/genesis.log" +$solana_genesis_demo < "$SOLANA_CONFIG_DIR"/mint-demo.json > "$SOLANA_CONFIG_DIR"/genesis.log + +echo "Creating $SOLANA_CONFIG_DIR/leader.json" +$solana_fullnode_config -d > "$SOLANA_CONFIG_DIR"/leader.json -cargo run --release --bin solana-mint-demo <<<"${num_tokens}" > mint-demo.json -cargo run --release --bin solana-genesis-demo < mint-demo.json > genesis.log +echo "Creating $SOLANA_CONFIG_DIR/validator.json" +$solana_fullnode_config -d -b 9000 > "$SOLANA_CONFIG_DIR"/validator.json -cargo run --release --bin solana-fullnode-config -- -d > leader-"${myip}".json -cargo run --release --bin solana-fullnode-config -- -b 9000 -d > validator-"${myip}".json +ls -lh "$SOLANA_CONFIG_DIR/" diff --git a/multinode-demo/validator.sh b/multinode-demo/validator.sh index 5bba1005bf078d..74e9f37a27733a 100755 --- a/multinode-demo/validator.sh +++ b/multinode-demo/validator.sh @@ -1,32 +1,50 @@ #!/bin/bash here=$(dirname "$0") +# shellcheck source=multinode-demo/common.sh +source "$here"/common.sh -# shellcheck source=/dev/null -. "${here}"/myip.sh - -leader=$1 - -[[ -z ${leader} ]] && { - echo "usage: $0 [network path to solana repo on leader machine]" +usage() { + echo "usage: $0 [network path to solana repo on leader machine] [network ip address of leader]" exit 1 } -myip=$(myip) || exit $? - -[[ -f validator-"$myip".json ]] || { - echo "I can't find a matching validator config file for \"${myip}\"... -Please run ${here}/setup.sh first. -" +if [[ "$1" = "-h" || -n "$3" ]]; then + usage +fi + +if [[ -d "$SNAP" ]]; then + # Select leader from the Snap configuration + leader_address="$(snapctl get leader-address)" + if [[ -z "$leader_address" ]]; then + # Assume drone is running on the same node as the leader by default + leader_address="localhost" + fi + leader=rsync://"$leader_address" +else + leader=${1:-${here}/..} # Default to local solana repo + leader_address=${2:-127.0.0.1} # Default to local leader +fi +leader_port=8001 + +if [[ -n "$SOLANA_CUDA" ]]; then + program="$solana_fullnode_cuda" +else + program="$solana_fullnode" +fi + + +[[ -f "$SOLANA_CONFIG_DIR"/validator.json ]] || { + echo "$SOLANA_CONFIG_DIR/validator.json not found, run ${here}/setup.sh first" exit 1 } -rsync -vz "${leader}"/{mint-demo.json,leader.json,genesis.log,tx-*.log} . || exit $? - -[[ $(uname) = Linux ]] && sudo sysctl -w net.core.rmem_max=26214400 1>/dev/null 2>/dev/null - -# if RUST_LOG is unset, default to info -export RUST_LOG=${RUST_LOG:-solana=info} +set -ex +SOLANA_LEADER_CONFIG_DIR="$SOLANA_CONFIG_DIR"/leader-config +rm -rf "$SOLANA_LEADER_CONFIG_DIR" +rsync -vPrz "${leader}"/config/ "$SOLANA_LEADER_CONFIG_DIR" +ls -lh "$SOLANA_LEADER_CONFIG_DIR" -cargo run --release --bin solana-fullnode -- \ - -l validator-"${myip}".json -v leader.json \ - < genesis.log tx-*.log +# shellcheck disable=SC2086 # $program should not be quoted +exec $program \ + -l "$SOLANA_CONFIG_DIR"/validator.json -t "$leader_address:$leader_port" \ + < "$SOLANA_LEADER_CONFIG_DIR"/genesis.log "$SOLANA_LEADER_CONFIG_DIR"/tx-*.log diff --git a/snap/hooks/configure b/snap/hooks/configure new file mode 100755 index 00000000000000..f82a6b98d89a3d --- /dev/null +++ b/snap/hooks/configure @@ -0,0 +1,30 @@ +#!/bin/bash -e + +echo Stopping daemons +snapctl stop --disable solana.daemon-drone +snapctl stop --disable solana.daemon-leader +snapctl stop --disable solana.daemon-validator + +mode="$(snapctl get mode)" +if [[ -z "$mode" ]]; then + exit 0 +fi + +$SNAP/bin/setup.sh "$(snapctl get num-tokens)" + +case $mode in +leader+drone) + snapctl start --enanble solana.daemon-leader + snapctl start --enanble solana.daemon-drone + ;; +leader) + snapctl start --enanble solana.daemon-leader + ;; +validator) + snapctl start --enanble solana.daemon-validator + ;; +*) + echo "Error: Unknown mode: $mode" + exit 1 + ;; +esac diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index fc789b6d187bab..3667dac142915e 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -10,6 +10,10 @@ grade: devel # CUDA dependency, so use 'devmode' confinement for now confinement: devmode +hooks: + configure: + plugs: [] + apps: drone: command: solana-drone @@ -44,7 +48,26 @@ apps: client-demo: command: solana-client-demo + daemon-validator: + daemon: simple + command: validator.sh + + daemon-leader: + daemon: simple + command: leader.sh + + daemon-drone: + daemon: simple + command: drone.sh + parts: + solana-scripts: + plugin: nil + override-build: | + set -x + mkdir -p $SNAPCRAFT_PART_INSTALL/bin + cp -av multinode-demo/* $SNAPCRAFT_PART_INSTALL/bin/ + solana-cuda: plugin: rust rust-channel: stable