From ce361c2cdc4db7445645b078b157ec31dc1a0e8c Mon Sep 17 00:00:00 2001
From: Michael Vines <mvines@gmail.com>
Date: Sun, 24 Jun 2018 10:10:55 -0700
Subject: [PATCH] Add Snap fullnode daemon

---
 .gitignore                  |  6 +++
 README.md                   | 84 ++++++++++++++++++++++++++++++++-----
 ci/snap.sh                  |  3 ++
 ci/upload_ci_artifact.sh    | 18 ++++++++
 multinode-demo/client.sh    | 29 +++++++------
 multinode-demo/common.sh    | 41 ++++++++++++++++++
 multinode-demo/drone.sh     | 34 +++++++++++++++
 multinode-demo/leader.sh    | 40 +++++++++---------
 multinode-demo/setup.sh     | 28 +++++++++----
 multinode-demo/validator.sh | 66 +++++++++++++++++++----------
 snap/hooks/configure        | 30 +++++++++++++
 snap/snapcraft.yaml         | 32 ++++++++++++++
 12 files changed, 339 insertions(+), 72 deletions(-)
 create mode 100644 ci/upload_ci_artifact.sh
 create mode 100644 multinode-demo/common.sh
 create mode 100755 multinode-demo/drone.sh
 create mode 100755 snap/hooks/configure

diff --git a/.gitignore b/.gitignore
index 3d4a8febfbe381..69790d143ea8e6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,9 @@ Cargo.lock
 /target/
 **/*.rs.bk
 .cargo
+
+# node configuration files
+/config/
+/config-drone/
+/config-validator/
+/config-client-demo/
diff --git a/README.md b/README.md
index 5555776165183e..9b43ea14182fae 100644
--- a/README.md
+++ b/README.md
@@ -84,13 +84,12 @@ Now start the server:
 $ ./multinode-demo/leader.sh
 ```
 
-To run a performance-enhanced fullnode on Linux, download `libcuda_verify_ed25519.a`. Enable
-it by adding `--features=cuda` to the line that runs `solana-fullnode` in
-`leader.sh`.  [CUDA 9.2](https://developer.nvidia.com/cuda-downloads) must be installed on your system.
-
+To run a performance-enhanced fullnode on Linux,
+[CUDA 9.2](https://developer.nvidia.com/cuda-downloads) must be installed on
+your system:
 ```bash
 $ ./fetch-perf-libs.sh
-$ cargo run --release --features=cuda --bin solana-fullnode -- -l leader.json < genesis.log
+$ SOLANA_CUDA=1 ./multinode-demo/leader.sh
 ```
 
 Wait a few seconds for the server to initialize. It will print "Ready." when it's ready to
@@ -102,14 +101,15 @@ Multinode Testnet
 To run a multinode testnet, after starting a leader node, spin up some validator nodes:
 
 ```bash
-$ ./multinode-demo/validator.sh ubuntu@10.0.1.51:~/solana #The leader machine
+$ ./multinode-demo/validator.sh ubuntu@10.0.1.51:~/solana 10.0.1.51
 ```
 
-As with the leader node, you can run a performance-enhanced validator fullnode by adding
-`--features=cuda` to the line that runs `solana-fullnode` in `validator.sh`.
-
+To run a performance-enhanced fullnode on Linux,
+[CUDA 9.2](https://developer.nvidia.com/cuda-downloads) must be installed on
+your system:
 ```bash
-$ cargo run --release --features=cuda --bin solana-fullnode -- -l validator.json -v leader.json < genesis.log
+$ ./fetch-perf-libs.sh
+$ SOLANA_CUDA=1 ./multinode-demo/leader.sh ubuntu@10.0.1.51:~/solana 10.0.1.51
 ```
 
 
@@ -152,6 +152,70 @@ $ snap info solana
 $ sudo snap refresh solana --devmode
 ```
 
+### Daemon support
+The snap supports running a leader, validator or leader+drone node as a system
+daemon.
+
+Run `sudo snap get solana` to view the current daemon configuration, and
+`sudo snap logs -f solana` to view the daemon logs.
+
+Disable the daemon at any time by running:
+```bash
+$ sudo snap set solana mode=
+```
+
+Runtime configuration files for the daemon can be found in
+`/var/snap/solana/current/config`.
+
+#### Leader daemon
+```bash
+$ sudo snap set solana mode=leader
+```
+
+If CUDA is available:
+```bash
+$ sudo snap set solana mode=leader enable-cuda=1
+```
+
+`rsync` must be configured and running on the leader.
+
+1. Ensure rsync is installed with `sudo apt-get -y install rsync`
+2. Edit `/etc/rsyncd.conf` to include the following
+```
+[config]
+path = /var/snap/solana/current/config
+hosts allow = *
+read only = true
+```
+3. Run `sudo systemctl enable rsync; sudo systemctl start rsync`
+4. Test by running `rsync -Pzravv rsync://<ip-address-of-leader>/config
+solana-config` from another machine.  If the leader is running on a cloud
+provider it may be necessary to configure the Firewall rules to permit ingress
+to port tcp:873, tcp:9900 and the port range udp:8000-udp:10000
+
+
+To run both the Leader and Drone:
+```bash
+$ sudo snap set solana mode=leader+drone
+```
+
+#### Validator daemon
+```bash
+$ sudo snap set solana mode=validator
+```
+If CUDA is available:
+```bash
+$ sudo snap set solana mode=validator enable-cuda=1
+```
+
+By default the validator will connect to **testnet.solana.com**, override
+the leader IP address by running:
+```bash
+$ sudo snap set solana mode=validator leader-address=127.0.0.1 #<-- change IP address
+```
+It's assumed that the leader will be running `rsync` configured as described in
+the previous **Leader daemon** section.
+
 Developing
 ===
 
diff --git a/ci/snap.sh b/ci/snap.sh
index c12a9f2801dff0..8d68c77ec93574 100755
--- a/ci/snap.sh
+++ b/ci/snap.sh
@@ -36,5 +36,8 @@ set -x
 echo --- build
 snapcraft
 
+source ci/upload_ci_artifact.sh
+upload_ci_artifact solana_*.snap
+
 echo --- publish
 $DRYRUN snapcraft push solana_*.snap --release $SNAP_CHANNEL
diff --git a/ci/upload_ci_artifact.sh b/ci/upload_ci_artifact.sh
new file mode 100644
index 00000000000000..9bbdce3c7369b0
--- /dev/null
+++ b/ci/upload_ci_artifact.sh
@@ -0,0 +1,18 @@
+# |source| me
+
+upload_ci_artifact() {
+  echo "--- artifact: $1"
+  if [[ -r "$1" ]]; then
+    ls -l "$1"
+    if ${BUILDKITE:-false}; then
+      (
+        set -x
+        buildkite-agent artifact upload "$1"
+      )
+    fi
+  else
+    echo ^^^ +++
+    echo "$1 not found"
+  fi
+}
+
diff --git a/multinode-demo/client.sh b/multinode-demo/client.sh
index ef69e6de4eeb6d..b358fe8892a1cf 100755
--- a/multinode-demo/client.sh
+++ b/multinode-demo/client.sh
@@ -1,17 +1,22 @@
 #!/bin/bash
+#
+# usage: $0 <network path to solana repo on leader machine> <number of nodes in the network>"
+#
 
-if [[ -z $1 ]]; then
-    echo "usage: $0 [network path to solana repo on leader machine] <number of nodes in the network>"
-    exit 1
-fi
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+SOLANA_CONFIG_DIR=config-client-demo
 
-LEADER=$1
-COUNT=${2:-1}
+leader=${1:-${here}/..}  # Default to local solana repo
+count=${2:-1}
 
-rsync -vz "$LEADER"/{leader.json,mint-demo.json} . || exit $?
+set -ex
+mkdir -p $SOLANA_CONFIG_DIR
+rsync -vz "$leader"/config/leader.json $SOLANA_CONFIG_DIR/
+rsync -vz "$leader"/config/mint-demo.json $SOLANA_CONFIG_DIR/
 
-# if RUST_LOG is unset, default to info
-export RUST_LOG=${RUST_LOG:-solana=info}
-
-cargo run --release --bin solana-client-demo -- \
-  -n "$COUNT" -l leader.json -d < mint-demo.json 2>&1 | tee client.log
+# shellcheck disable=SC2086 # $solana_client_demo should not be quoted
+exec $solana_client_demo \
+  -n "$count" -l $SOLANA_CONFIG_DIR/leader.json -d \
+  < $SOLANA_CONFIG_DIR/mint-demo.json 2>&1 | tee client-demo.log
diff --git a/multinode-demo/common.sh b/multinode-demo/common.sh
new file mode 100644
index 00000000000000..4d75f7edaa9b6a
--- /dev/null
+++ b/multinode-demo/common.sh
@@ -0,0 +1,41 @@
+# |source| this file
+#
+# Disable complaints about unused variables in this file:
+# shellcheck disable=2034
+
+if [[ -d "$SNAP" ]]; then # Running as a Linux Snap?
+  solana_program() {
+    local program="$1"
+    printf "%s/command-%s.wrapper" "$SNAP" "$program"
+  }
+  SOLANA_CUDA="$(snapctl get enable-cuda)"
+elif [[ -n "$USE_INSTALL" ]]; then # Assume |cargo install| was run
+  solana_program() {
+    local program="$1"
+    printf "solana-%s" "$program"
+  }
+else
+  solana_program() {
+    local program="$1"
+    local features=""
+    if [[ "$program" =~ ^(.*)-cuda$ ]]; then
+      program=${BASH_REMATCH[1]}
+      features="--features=cuda,erasure"
+    fi
+    printf "cargo run --release --bin solana-%s %s -- " "$program" "$features"
+  }
+fi
+
+solana_client_demo=$(solana_program client-demo)
+solana_drone=$(solana_program drone)
+solana_fullnode=$(solana_program fullnode)
+solana_fullnode_config=$(solana_program fullnode-config)
+solana_fullnode_cuda=$(solana_program fullnode-cuda)
+solana_genesis_demo=$(solana_program genesis-demo)
+solana_mint_demo=$(solana_program mint-demo)
+
+export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
+export RUST_BACKTRACE=1
+[[ $(uname) = Linux ]] && (set -x; sudo sysctl -w net.core.rmem_max=26214400 1>/dev/null 2>/dev/null)
+
+SOLANA_CONFIG_DIR=${SNAP_DATA:-$PWD}/config
diff --git a/multinode-demo/drone.sh b/multinode-demo/drone.sh
new file mode 100755
index 00000000000000..07e3d1863a291c
--- /dev/null
+++ b/multinode-demo/drone.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# usage: $0 <network path to solana repo on leader machine>
+#
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+SOLANA_CONFIG_DIR=config-drone
+
+if [[ -d "$SNAP" ]]; then
+  # Exit if mode is not yet configured
+  # (typically the case after the Snap is first installed)
+  [[ -n "$(snapctl get mode)" ]] || exit 0
+
+  # Select leader from the Snap configuration
+  leader_address="$(snapctl get leader-address)"
+  if [[ -z "$leader_address" ]]; then
+    # Assume drone is running on the same node as the leader by default
+    leader_address="localhost"
+  fi
+  leader=rsync://"$leader_address"
+else
+  leader=${1:-${here}/..}  # Default to local solana repo
+fi
+
+set -ex
+mkdir -p $SOLANA_CONFIG_DIR
+rsync -vz "$leader"/config/leader.json $SOLANA_CONFIG_DIR/
+rsync -vz "$leader"/config/mint-demo.json $SOLANA_CONFIG_DIR/
+
+# shellcheck disable=SC2086 # $solana_drone should not be quoted
+exec $solana_drone \
+  -l $SOLANA_CONFIG_DIR/leader.json < $SOLANA_CONFIG_DIR/mint-demo.json
diff --git a/multinode-demo/leader.sh b/multinode-demo/leader.sh
index 6d9e5a098e9f75..733f0680862b2c 100755
--- a/multinode-demo/leader.sh
+++ b/multinode-demo/leader.sh
@@ -1,28 +1,28 @@
 #!/bin/bash
-here=$(dirname "$0")
 
-# shellcheck source=/dev/null
-. "${here}"/myip.sh
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
 
-myip=$(myip) || exit $?
+if [[ -d "$SNAP" ]]; then
+  # Exit if mode is not yet configured
+  # (typically the case after the Snap is first installed)
+  [[ -n "$(snapctl get mode)" ]] || exit 0
+fi
 
-[[ -f leader-"${myip}".json ]] || {
-  echo "I can't find a matching leader config file for \"${myip}\"...
-Please run ${here}/setup.sh first.
-"
+[[ -f "$SOLANA_CONFIG_DIR"/leader.json ]] || {
+  echo "$SOLANA_CONFIG_DIR/leader.json not found, run ${here}/setup.sh first"
   exit 1
 }
 
-# if RUST_LOG is unset, default to info
-export RUST_LOG=${RUST_LOG:-solana=info}
-
-[[ $(uname) = Linux ]] && sudo sysctl -w net.core.rmem_max=26214400 1>/dev/null 2>/dev/null
-
-# this makes a leader.json file available alongside genesis, etc. for
-#  validators and clients
-cp leader-"${myip}".json leader.json
+if [[ -n "$SOLANA_CUDA" ]]; then
+  program="$solana_fullnode_cuda"
+else
+  program="$solana_fullnode"
+fi
 
-cargo run --release --bin solana-fullnode -- \
-      -l leader-"${myip}".json \
-      < genesis.log tx-*.log \
-      > tx-"$(date -u +%Y%m%d%H%M%S%N)".log
+# shellcheck disable=SC2086 # $program should not be quoted
+exec $program \
+  -l "$SOLANA_CONFIG_DIR"/leader.json \
+  < "$SOLANA_CONFIG_DIR"/genesis.log "$SOLANA_CONFIG_DIR"/tx-*.log \
+  > "$SOLANA_CONFIG_DIR"/tx-"$(date -u +%Y%m%d%H%M%S%N)".log
diff --git a/multinode-demo/setup.sh b/multinode-demo/setup.sh
index 3c68abccab1159..c6c347e96b4869 100755
--- a/multinode-demo/setup.sh
+++ b/multinode-demo/setup.sh
@@ -1,15 +1,27 @@
 #!/bin/bash
+
+num_tokens=${1:-1000000000}
+
 here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
 
-# shellcheck source=/dev/null
-. "${here}"/myip.sh
+set -e
 
-myip=$(myip) || exit $?
+echo "Cleaning $SOLANA_CONFIG_DIR"
+rm -rvf "$SOLANA_CONFIG_DIR"
+mkdir -p "$SOLANA_CONFIG_DIR"
 
-num_tokens=${1:-1000000000}
+echo "Creating $SOLANA_CONFIG_DIR/mint-demo.json with $num_tokens tokens"
+$solana_mint_demo <<<"$num_tokens" > "$SOLANA_CONFIG_DIR"/mint-demo.json
+
+echo "Creating $SOLANA_CONFIG_DIR/genesis.log"
+$solana_genesis_demo < "$SOLANA_CONFIG_DIR"/mint-demo.json > "$SOLANA_CONFIG_DIR"/genesis.log
+
+echo "Creating $SOLANA_CONFIG_DIR/leader.json"
+$solana_fullnode_config -d > "$SOLANA_CONFIG_DIR"/leader.json
 
-cargo run --release --bin solana-mint-demo <<<"${num_tokens}" > mint-demo.json
-cargo run --release --bin solana-genesis-demo < mint-demo.json > genesis.log
+echo "Creating $SOLANA_CONFIG_DIR/validator.json"
+$solana_fullnode_config -d -b 9000 > "$SOLANA_CONFIG_DIR"/validator.json
 
-cargo run --release --bin solana-fullnode-config -- -d > leader-"${myip}".json
-cargo run --release --bin solana-fullnode-config -- -b 9000 -d > validator-"${myip}".json
+ls -lh "$SOLANA_CONFIG_DIR/"
diff --git a/multinode-demo/validator.sh b/multinode-demo/validator.sh
index 5bba1005bf078d..b19d34c5b47ed2 100755
--- a/multinode-demo/validator.sh
+++ b/multinode-demo/validator.sh
@@ -1,32 +1,54 @@
 #!/bin/bash
 here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
 
-# shellcheck source=/dev/null
-. "${here}"/myip.sh
-
-leader=$1
-
-[[ -z ${leader} ]] && {
-  echo "usage: $0 [network path to solana repo on leader machine]"
+usage() {
+  echo "usage: $0 [network path to solana repo on leader machine] [network ip address of leader]"
   exit 1
 }
 
-myip=$(myip) || exit $?
-
-[[ -f validator-"$myip".json ]] || {
-  echo "I can't find a matching validator config file for \"${myip}\"...
-Please run ${here}/setup.sh first.
-"
+if [[ "$1" = "-h" || -n "$3" ]]; then
+  usage
+fi
+
+if [[ -d "$SNAP" ]]; then
+  # Exit if mode is not yet configured
+  # (typically the case after the Snap is first installed)
+  [[ -n "$(snapctl get mode)" ]] || exit 0
+
+  # Select leader from the Snap configuration
+  leader_address="$(snapctl get leader-address)"
+  if [[ -z "$leader_address" ]]; then
+    # Assume drone is running on the same node as the leader by default
+    leader_address="localhost"
+  fi
+  leader=rsync://"$leader_address"
+else
+  leader=${1:-${here}/..}    # Default to local solana repo
+  leader_address=${2:-127.0.0.1}  # Default to local leader
+fi
+leader_port=8001
+
+if [[ -n "$SOLANA_CUDA" ]]; then
+  program="$solana_fullnode_cuda"
+else
+  program="$solana_fullnode"
+fi
+
+
+[[ -f "$SOLANA_CONFIG_DIR"/validator.json ]] || {
+  echo "$SOLANA_CONFIG_DIR/validator.json not found, run ${here}/setup.sh first"
   exit 1
 }
 
-rsync -vz "${leader}"/{mint-demo.json,leader.json,genesis.log,tx-*.log} . || exit $?
-
-[[ $(uname) = Linux ]] && sudo sysctl -w net.core.rmem_max=26214400 1>/dev/null 2>/dev/null
-
-# if RUST_LOG is unset, default to info
-export RUST_LOG=${RUST_LOG:-solana=info}
+set -ex
+SOLANA_LEADER_CONFIG_DIR="$SOLANA_CONFIG_DIR"/leader-config
+rm -rf "$SOLANA_LEADER_CONFIG_DIR"
+rsync -vPrz "${leader}"/config/ "$SOLANA_LEADER_CONFIG_DIR"
+ls -lh "$SOLANA_LEADER_CONFIG_DIR"
 
-cargo run --release --bin solana-fullnode -- \
-      -l validator-"${myip}".json -v leader.json \
-      < genesis.log tx-*.log
+# shellcheck disable=SC2086 # $program should not be quoted
+exec $program \
+  -l "$SOLANA_CONFIG_DIR"/validator.json -t "$leader_address:$leader_port" \
+  < "$SOLANA_LEADER_CONFIG_DIR"/genesis.log "$SOLANA_LEADER_CONFIG_DIR"/tx-*.log
diff --git a/snap/hooks/configure b/snap/hooks/configure
new file mode 100755
index 00000000000000..fce0df82918d63
--- /dev/null
+++ b/snap/hooks/configure
@@ -0,0 +1,30 @@
+#!/bin/bash -e
+
+echo Stopping daemons
+snapctl stop --disable solana.daemon-drone
+snapctl stop --disable solana.daemon-leader
+snapctl stop --disable solana.daemon-validator
+
+mode="$(snapctl get mode)"
+if [[ -z "$mode" ]]; then
+  exit 0
+fi
+
+$SNAP/bin/setup.sh "$(snapctl get num-tokens)"
+
+case $mode in
+leader+drone)
+  snapctl start --enable solana.daemon-leader
+  snapctl start --enable solana.daemon-drone
+  ;;
+leader)
+  snapctl start --enable solana.daemon-leader
+  ;;
+validator)
+  snapctl start --enable solana.daemon-validator
+  ;;
+*)
+  echo "Error: Unknown mode: $mode"
+  exit 1
+  ;;
+esac
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index fc789b6d187bab..c1c82061c5ac9b 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -10,6 +10,10 @@ grade: devel
 # CUDA dependency, so use 'devmode' confinement for now
 confinement: devmode
 
+hooks:
+  configure:
+    plugs: []
+
 apps:
   drone:
     command: solana-drone
@@ -44,7 +48,35 @@ apps:
   client-demo:
     command: solana-client-demo
 
+  daemon-validator:
+    daemon: simple
+    command: validator.sh
+
+  daemon-leader:
+    daemon: simple
+    command: leader.sh
+
+  daemon-drone:
+    daemon: simple
+    command: drone.sh
+
 parts:
+  solana-rsync:
+    plugin: nil
+    override-build: |
+      # TODO: build rsync from source instead of sneaking it in from the host
+      # system...
+      set -x
+      mkdir -p $SNAPCRAFT_PART_INSTALL/bin
+      cp -av /usr/bin/rsync $SNAPCRAFT_PART_INSTALL/bin/
+
+  solana-scripts:
+    plugin: nil
+    override-build: |
+      set -x
+      mkdir -p $SNAPCRAFT_PART_INSTALL/bin
+      cp -av multinode-demo/* $SNAPCRAFT_PART_INSTALL/bin/
+
   solana-cuda:
     plugin: rust
     rust-channel: stable