added dummy signer address; checked for primary validator status

accounts/keystore/69103d3decf4a462e7f95b36684b04f86a950028.json

@@ -0,0 +1 @@
+{"id":"4413e85c-3653-215d-b842-2c0b1e702e76","version":3,"crypto":{"cipher":"aes-128-ctr","cipherparams":{"iv":"16d746e0b9f340bcfecf0f48bfa1a58c"},"ciphertext":"00d5145dc0647fd1869ad6ba7e41c1dfe4d31ff627029bd2d9352f20c9da8e94","kdf":"pbkdf2","kdfparams":{"c":10240,"dklen":32,"prf":"hmac-sha256","salt":"aae3a89625a7c9a506488fd087a0c17b1cd743c9584fd97b924ff9b9824981b1"},"mac":"c39f1923a75cf3829669a8e3a0fd25e6c6d702aef45d7f2b3e9f12b6f08d07b4"},"address":"69103d3decf4a462e7f95b36684b04f86a950028","name":"","meta":"{}"}

parity-data/node6/keys/DPoSChain/dummy-signer.json

@@ -0,0 +1 @@
+{"id":"4413e85c-3653-215d-b842-2c0b1e702e76","version":3,"crypto":{"cipher":"aes-128-ctr","cipherparams":{"iv":"16d746e0b9f340bcfecf0f48bfa1a58c"},"ciphertext":"00d5145dc0647fd1869ad6ba7e41c1dfe4d31ff627029bd2d9352f20c9da8e94","kdf":"pbkdf2","kdfparams":{"c":10240,"dklen":32,"prf":"hmac-sha256","salt":"aae3a89625a7c9a506488fd087a0c17b1cd743c9584fd97b924ff9b9824981b1"},"mac":"c39f1923a75cf3829669a8e3a0fd25e6c6d702aef45d7f2b3e9f12b6f08d07b4"},"address":"69103d3decf4a462e7f95b36684b04f86a950028","name":"","meta":"{}"}

scripts/stop-test-setup

@@ -11,6 +11,11 @@ function kill_at_port {
 }
 
 function kill_at_file {
+    if [[ ! -a $1 ]]; then
+        echo "No such file: "$1
+        return
+    fi
+
     PID=`lsof -t ${1}`
     if [ "$PID" != "" ]; then
         echo Killing pid $PID at file $1

scripts/watchBackedUpNode.js

@@ -1,31 +1,48 @@
+// Validator node failover script
+//
+// The script should be run on the secondary (failover) node that backs up a
+// primary node. The secondary node has signing disabled by default. It starts
+// signing blocks when the primary node is a validator but has not produced a
+// block when it was its turn and this script cannot connect to it by HTTP. The
+// secondary node stops sigining blocks as soon as the HTTP connection to the
+// primary node re-establishes.
+
 var assert = require("assert");
 var fs = require("fs");
 var { promisify } = require("util");
 var readFile = promisify(fs.readFile);
 var ethers = require("ethers");
 
-const SIGNER_ADDRESS = "0x522df396ae70a058bd69778408630fdb023389b2";
 const PORT1 = "8543";
 const PORT2 = "8546";
 const RETRY_TIMEOUT_SECONDS = 2;
 const SCAN_INTERVAL_SECONDS = 5;
-const MAX_VALIDATOR_SET_SIZE = 21;
+const ARTIFACTS_PATH = "../posdao-contracts/build/contracts/";
+const PASSWORD_PATH = "/../config/password"
+const SIGNER_ADDRESS1 = "0x522df396ae70a058bd69778408630fdb023389b2";
+const SIGNER_ADDRESS2 = "0x522df396ae70a058bd69778408630fdb023389b2";
+const DUMMY_SIGNER_ADDRESS = "0x69103d3decf4a462e7f95b36684b04f86a950028";
 
 var Web3 = require("web3");
-var web3_1 = new Web3(new Web3.providers.HttpProvider("http://localhost:" + PORT1));
-var web3_2 = new Web3(new Web3.providers.HttpProvider("http://localhost:" + PORT2));
-var provider = new ethers.providers.JsonRpcProvider("http://localhost:" + PORT2);
+var web3_1 = new Web3(new Web3.providers.HttpProvider(`http://localhost:${PORT1}`));
+var web3_2 = new Web3(new Web3.providers.HttpProvider(`http://localhost:${PORT2}`));
+var provider = new ethers.providers.JsonRpcProvider(`http://localhost:${PORT2}`);
 // `true` if the primary is required to sign and `false` if the secondary does.
 var primaryHasToSign = true;
+var validatorSetContract = new web3_2.eth.Contract(
+  require(`${ARTIFACTS_PATH}ValidatorSetAuRa.json`).abi,
+  '0x1000000000000000000000000000000000000001'
+);
 
-async function scanBlocks() {
+async function scanBlocks(depth) {
+    assert(typeof depth === "number");
     var lastBlock = await web3_2.eth.getBlock("latest");
     var lastBlockNum = lastBlock.number;
     assert(typeof lastBlockNum === "number");
-    if (lastBlockNum < MAX_VALIDATOR_SET_SIZE) {
+    if (lastBlockNum < depth) {
         return true;
     }
-    var startBlockNum = lastBlockNum - MAX_VALIDATOR_SET_SIZE;
+    var startBlockNum = lastBlockNum - depth;
 
     console.log(`Scanning blocks from ${startBlockNum} to ${lastBlockNum}`);
 
@@ -38,23 +55,27 @@ async function scanBlocks() {
     return false;
 }
 
+// Starts signing at the secondary node by setting the secondary signer address.
 async function startSecondarySigning() {
-    console.log("Reserve node at port " + PORT + " starts signing");
+    console.log(`Reserve node at port ${PORT2} starts signing`);
 
-    let password = await readFile(__dirname + "/../config/password", "UTF-8");
+    let password = await readFile(`__dirname${PASSWORD_PATH}`, "UTF-8");
     assert(typeof password === "string");
     await provider.send(
         "parity_setEngineSigner",
-        [ SIGNER_ADDRESS, password.trim() ]
+        [ SIGNER_ADDRESS2, password.trim() ]
     );
 }
 
+// Stops signing at the secondary node by setting the dummy signer address.
 async function stopSecondarySigning() {
-    console.log("Reserve node at port " + PORT + " stops signing");
+    console.log(`Reserve node at port ${PORT2} stops signing`);
 
+    let password = await readFile(`__dirname${PASSWORD_PATH}`, "UTF-8");
+    assert(typeof password === "string");
     await provider.send(
         "parity_setEngineSigner",
-        [ "0x0000000000000000000000000000000000000000", "" ]
+        [ DUMMY_SIGNER_ADDRESS, password.trim() ]
     );
 }
 
@@ -67,34 +88,41 @@ async function startScan() {
     }
     assert(typeof secondaryListening === "boolean");
     if (secondaryListening) {
-        var primaryListening = false;
-        try {
-            primaryListening = await web3_1.eth.net.isListening();
-        } catch(e) {
-            console.log("Disconnected from primary");
-        }
-        assert(typeof primaryListening === "boolean");
-        if (!primaryListening) {
-            let signed = await scanBlocks();
-            if (!signed) {
-                console.log(`Failed to find a block authored by ${SIGNER_ADDRESS}`);
-                if (primaryHasToSign) {
-                    primaryHasToSign = false;
-                    await startSecondarySigning();
-                }
+        let validators = await validatorSetContract.methods.getValidators().call();
+        // Perform failover checks only if the primary is currently a validator.
+        if (validators.indexOf(SIGNER_ADDRESS1) != -1) {
+            var primaryListening = false;
+            try {
+                primaryListening = await web3_1.eth.net.isListening();
+            } catch(e) {
+                console.log("Disconnected from primary");
             }
-        } else {
-            if (!primaryHasToSign) {
-                console.log("Primary has come back up; moving the secondary to reserve");
-                primaryHasToSign = true;
-                await stopSecondarySigning();
+            assert(typeof primaryListening === "boolean");
+            if (!primaryListening) {
+                // Ensure that we (the secondary mode) are still connected to the
+                // network by checking that other validators continued to sign
+                // blocks.
+                let signed = await scanBlocks(validators.length);
+                if (!signed) {
+                    console.log(`Failed to find a block authored by ${SIGNER_ADDRESS}`);
+                    if (primaryHasToSign) {
+                        primaryHasToSign = false;
+                        await startSecondarySigning();
+                    }
+                }
+            } else {
+                if (!primaryHasToSign) {
+                    console.log("Primary has come back up; moving the secondary to reserve");
+                    primaryHasToSign = true;
+                    await stopSecondarySigning();
+                }
             }
         }
+        setTimeout(startScan, SCAN_INTERVAL_SECONDS * 1000);
     } else {
         console.log(`Disconnected from secondary; retry in ${RETRY_TIMEOUT_SECONDS}s`);
-        return setTimeout(startScan, RETRY_TIMEOUT_SECONDS * 1000);
+        setTimeout(startScan, RETRY_TIMEOUT_SECONDS * 1000);
     }
-    setTimeout(startScan, SCAN_INTERVAL_SECONDS * 1000);
 }
 
 try {