From 5b5fbe0a8f8c41a27041d54fe3efff9542dba168 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 3 Jun 2020 12:17:58 -0500 Subject: [PATCH 01/12] Removed passing in nodeId to Node methods and added it to __init__. --- tests/Cluster.py | 30 ++++++++++++----- tests/Node.py | 33 ++++++++++--------- tests/block_log_util_test.py | 4 +-- tests/nodeos_chainbase_allocation_test.py | 2 +- tests/nodeos_forked_chain_test.py | 2 +- tests/nodeos_irreversible_mode_test.py | 28 ++++++++-------- ..._multiple_version_protocol_feature_test.py | 18 +++++----- tests/nodeos_protocol_feature_test.py | 8 ++--- tests/nodeos_startup_catchup.py | 2 +- 9 files changed, 70 insertions(+), 57 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index a59e4f3d03..ca091f23c4 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -87,6 +87,7 @@ def __init__(self, walletd=False, localCluster=True, host="localhost", port=8888 self.useBiosBootFile=False self.filesToCleanup=[] self.alternateVersionLabels=Cluster.__defaultAlternateVersionLabels() + self.biosNode = None def setChainStrategy(self, chainSyncStrategy=Utils.SyncReplayTag): @@ -464,7 +465,8 @@ def initAccountKeys(account, keys): def initializeNodes(self, defproduceraPrvtKey=None, defproducerbPrvtKey=None, onlyBios=False): port=Cluster.__BiosPort if onlyBios else self.port host=Cluster.__BiosHost if onlyBios else self.host - node=Node(host, port, walletMgr=self.walletMgr) + nodeNum="bios" if onlyBios else 0 + node=Node(host, port, nodeNum, walletMgr=self.walletMgr) if Utils.Debug: Utils.Print("Node: %s", str(node)) node.checkPulse(exitOnError=True) @@ -503,12 +505,13 @@ def initializeNodesFromJson(self, nodesJsonStr): for n in nArr: port=n["port"] host=n["host"] - node=Node(host, port, walletMgr=self.walletMgr) + node=Node(host, port, nodeId=len(nodes), walletMgr=self.walletMgr) if Utils.Debug: Utils.Print("Node:", node) node.checkPulse(exitOnError=True) nodes.append(node) + self.nodes=nodes return True @@ -669,7 +672,17 @@ def getNode(self, nodeId=0, exitOnError=True): return self.nodes[nodeId] def getNodes(self): - return self.nodes + nodes = [] + if hasattr(self, "nodes") and self.nodes is not None: + nodes += self.nodes + return nodes + + def getAllNodes(self): + nodes = [] + if self.biosNode is not None: + nodes.append(self.biosNode) + nodes += self.getNodes() + return nodes def launchUnstarted(self, numToLaunch=1, cachePopen=False): assert(isinstance(numToLaunch, int)) @@ -678,7 +691,7 @@ def launchUnstarted(self, numToLaunch=1, cachePopen=False): del self.unstartedNodes[:numToLaunch] for node in launchList: # the node number is indexed off of the started nodes list - node.launchUnstarted(len(self.nodes), cachePopen=cachePopen) + node.launchUnstarted(cachePopen=cachePopen) self.nodes.append(node) # Spread funds across accounts with transactions spread through cluster nodes. @@ -1342,7 +1355,7 @@ def discoverLocalNode(self, nodeNum, psOut=None, timeout=None): if m is None: Utils.Print("ERROR: Failed to find %s pid. Pattern %s" % (Utils.EosServerName, pattern)) return None - instance=Node(self.host, self.port + nodeNum, pid=int(m.group(1)), cmd=m.group(2), walletMgr=self.walletMgr) + instance=Node(self.host, self.port + nodeNum, nodeNum, pid=int(m.group(1)), cmd=m.group(2), walletMgr=self.walletMgr) if Utils.Debug: Utils.Print("Node>", instance) return instance @@ -1355,7 +1368,7 @@ def discoverBiosNode(self, timeout=None): Utils.Print("ERROR: Failed to find %s pid. Pattern %s" % (Utils.EosServerName, pattern)) return None else: - return Node(Cluster.__BiosHost, Cluster.__BiosPort, pid=int(m.group(1)), cmd=m.group(2), walletMgr=self.walletMgr) + return Node(Cluster.__BiosHost, Cluster.__BiosPort, "bios", pid=int(m.group(1)), cmd=m.group(2), walletMgr=self.walletMgr) # Kills a percentange of Eos instances starting from the tail and update eosInstanceInfos state def killSomeEosInstances(self, killCount, killSignalStr=Utils.SigKillTag): @@ -1383,7 +1396,7 @@ def relaunchEosInstances(self, cachePopen=False): newChain= False if self.__chainSyncStrategy.name in [Utils.SyncHardReplayTag, Utils.SyncNoneTag] else True for i in range(0, len(self.nodes)): node=self.nodes[i] - if node.killed and not node.relaunch(i, chainArg, newChain=newChain, cachePopen=cachePopen): + if node.killed and not node.relaunch(chainArg, newChain=newChain, cachePopen=cachePopen): return False return True @@ -1533,8 +1546,7 @@ def discoverUnstartedLocalNode(self, nodeId): with open(startFile, 'r') as file: cmd=file.read() Utils.Print("unstarted local node cmd: %s" % (cmd)) - p=re.compile(r'^\s*(\w+)\s*=\s*([^\s](?:.*[^\s])?)\s*$') - instance=Node(self.host, port=self.port+nodeId, pid=None, cmd=cmd, walletMgr=self.walletMgr) + instance=Node(self.host, port=self.port+nodeId, nodeId=nodeId, pid=None, cmd=cmd, walletMgr=self.walletMgr) if Utils.Debug: Utils.Print("Unstarted Node>", instance) return instance diff --git a/tests/Node.py b/tests/Node.py index 7b18eeaa68..6152ad6aca 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -4,10 +4,11 @@ import time import os import re -import datetime import json import signal +from datetime import datetime +from datetime import timedelta from core_symbol import CORE_SYMBOL from testUtils import Utils from testUtils import Account @@ -27,11 +28,14 @@ class Node(object): # pylint: disable=too-many-instance-attributes # pylint: disable=too-many-arguments - def __init__(self, host, port, pid=None, cmd=None, walletMgr=None): + def __init__(self, host, port, nodeId, pid=None, cmd=None, walletMgr=None): self.host=host self.port=port self.pid=pid self.cmd=cmd + if nodeId != "bios": + assert isinstance(nodeId, int) + self.nodeId=nodeId if Utils.Debug: Utils.Print("new Node host=%s, port=%s, pid=%s, cmd=%s" % (self.host, self.port, self.pid, self.cmd)) self.killed=False # marks node as killed self.endpointHttp="http://%s:%d" % (self.host, self.port) @@ -50,8 +54,7 @@ def eosClientArgs(self): return self.endpointArgs + walletArgs + " " + Utils.MiscEosClientArgs def __str__(self): - #return "Host: %s, Port:%d, Pid:%s, Cmd:\"%s\"" % (self.host, self.port, self.pid, self.cmd) - return "Host: %s, Port:%d, Pid:%s" % (self.host, self.port, self.pid) + return "Host: %s, Port:%d, NodeNum:%s, Pid:%s" % (self.host, self.port, self.nodeId, self.pid) @staticmethod def validateTransaction(trans): @@ -1087,16 +1090,14 @@ def getNextCleanProductionCycle(self, trans): return blockNum - # TBD: make nodeId an internal property # pylint: disable=too-many-locals # If nodeosPath is equal to None, it will use the existing nodeos path - def relaunch(self, nodeId, chainArg=None, newChain=False, skipGenesis=True, timeout=Utils.systemWaitTimeout, addSwapFlags=None, cachePopen=False, nodeosPath=None): + def relaunch(self, chainArg=None, newChain=False, skipGenesis=True, timeout=Utils.systemWaitTimeout, addSwapFlags=None, cachePopen=False, nodeosPath=None): assert(self.pid is None) assert(self.killed) - assert isinstance(nodeId, int) or (isinstance(nodeId, str) and nodeId == "bios"), "Invalid Node ID is passed" - if Utils.Debug: Utils.Print("Launching node process, Id: {}".format(nodeId)) + if Utils.Debug: Utils.Print("Launching node process, Id: {}".format(self.nodeId)) cmdArr=[] splittedCmd=self.cmd.split() @@ -1130,7 +1131,7 @@ def relaunch(self, nodeId, chainArg=None, newChain=False, skipGenesis=True, time myCmd=" ".join(cmdArr) cmd=myCmd + ("" if chainArg is None else (" " + chainArg)) - self.launchCmd(cmd, nodeId, cachePopen) + self.launchCmd(cmd, cachePopen) def isNodeAlive(): """wait for node to be responsive.""" @@ -1164,13 +1165,13 @@ def unstartedFile(nodeId): Utils.errorExit("Cannot find unstarted node since %s file does not exist" % startFile) return startFile - def launchUnstarted(self, nodeId, cachePopen=False): + def launchUnstarted(self, cachePopen=False): Utils.Print("launchUnstarted cmd: %s" % (self.cmd)) - self.launchCmd(self.cmd, nodeId, cachePopen) + self.launchCmd(self.cmd, cachePopen) - def launchCmd(self, cmd, nodeId, cachePopen=False): - dataDir=Utils.getNodeDataDir(nodeId) - dt = datetime.datetime.now() + def launchCmd(self, cmd, cachePopen=False): + dataDir=Utils.getNodeDataDir(self.nodeId) + dt = datetime.now() dateStr=Utils.getDateString(dt) stdoutFile="%s/stdout.%s.txt" % (dataDir, dateStr) stderrFile="%s/stderr.%s.txt" % (dataDir, dateStr) @@ -1310,8 +1311,8 @@ def getActivatedProtocolFeatures(self): latestBlockHeaderState = self.getLatestBlockHeaderState() return latestBlockHeaderState["activated_protocol_features"]["protocol_features"] - def modifyBuiltinPFSubjRestrictions(self, nodeId, featureCodename, subjectiveRestriction={}): - jsonPath = os.path.join(Utils.getNodeConfigDir(nodeId), + def modifyBuiltinPFSubjRestrictions(self, featureCodename, subjectiveRestriction={}): + jsonPath = os.path.join(Utils.getNodeConfigDir(self.nodeId), "protocol_features", "BUILTIN-{}.json".format(featureCodename)) protocolFeatureJson = [] diff --git a/tests/block_log_util_test.py b/tests/block_log_util_test.py index 1278808c50..86e470fff0 100755 --- a/tests/block_log_util_test.py +++ b/tests/block_log_util_test.py @@ -156,7 +156,7 @@ def checkBlockLog(blockLog, blockNumsToFind, firstBlockNum=1): # relaunch the node with the truncated block log and ensure it catches back up with the producers current_head_block_num = node1.getInfo()["head_block_num"] - cluster.getNode(2).relaunch(2, cachePopen=True) + cluster.getNode(2).relaunch(cachePopen=True) assert cluster.getNode(2).waitForBlock(current_head_block_num, timeout=60, reportInterval=15) # ensure it continues to advance @@ -186,7 +186,7 @@ def checkBlockLog(blockLog, blockNumsToFind, firstBlockNum=1): # relaunch the node with the truncated block log and ensure it catches back up with the producers current_head_block_num = node1.getInfo()["head_block_num"] assert current_head_block_num >= info["head_block_num"] - cluster.getNode(2).relaunch(2, cachePopen=True) + cluster.getNode(2).relaunch(cachePopen=True) assert cluster.getNode(2).waitForBlock(current_head_block_num, timeout=60, reportInterval=15) # ensure it continues to advance diff --git a/tests/nodeos_chainbase_allocation_test.py b/tests/nodeos_chainbase_allocation_test.py index 697ada3551..c5f2bbe2d2 100755 --- a/tests/nodeos_chainbase_allocation_test.py +++ b/tests/nodeos_chainbase_allocation_test.py @@ -102,7 +102,7 @@ def isSetProdsBlockNumIrr(): # Restart irr node and ensure the snapshot is still identical irrNode.kill(signal.SIGTERM) - isRelaunchSuccess = irrNode.relaunch(irrNodeId, "", timeout=5, cachePopen=True) + isRelaunchSuccess = irrNode.relaunch(timeout=5, cachePopen=True) assert isRelaunchSuccess, "Fail to relaunch" res = irrNode.createSnapshot() afterShutdownSnapshotPath = res["snapshot_name"] diff --git a/tests/nodeos_forked_chain_test.py b/tests/nodeos_forked_chain_test.py index a8193d39dc..54fa9c1b2c 100755 --- a/tests/nodeos_forked_chain_test.py +++ b/tests/nodeos_forked_chain_test.py @@ -478,7 +478,7 @@ def getBlock(self, blockNum): Print("Relaunching the non-producing bridge node to connect the producing nodes again") - if not nonProdNode.relaunch(nonProdNode.nodeNum, None): + if not nonProdNode.relaunch(): errorExit("Failure - (non-production) node %d should have restarted" % (nonProdNode.nodeNum)) diff --git a/tests/nodeos_irreversible_mode_test.py b/tests/nodeos_irreversible_mode_test.py index f511132ab1..e7caf14ec4 100755 --- a/tests/nodeos_irreversible_mode_test.py +++ b/tests/nodeos_irreversible_mode_test.py @@ -138,8 +138,8 @@ def confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest, headLibAndForkDbHeadBefore assert forkDbHead == forkDbHeadBeforeSwitchMode, \ "Fork db head ({}) should be equal to fork db head before switch mode ({}) ".format(forkDbHead, forkDbHeadBeforeSwitchMode) -def relaunchNode(node: Node, nodeId, chainArg="", addSwapFlags=None, relaunchAssertMessage="Fail to relaunch"): - isRelaunchSuccess = node.relaunch(nodeId, chainArg=chainArg, addSwapFlags=addSwapFlags, timeout=relaunchTimeout, cachePopen=True) +def relaunchNode(node: Node, chainArg="", addSwapFlags=None, relaunchAssertMessage="Fail to relaunch"): + isRelaunchSuccess = node.relaunch(chainArg=chainArg, addSwapFlags=addSwapFlags, timeout=relaunchTimeout, cachePopen=True) time.sleep(1) # Give a second to replay or resync if needed assert isRelaunchSuccess, relaunchAssertMessage return isRelaunchSuccess @@ -174,7 +174,7 @@ def stopProdNode(): def startProdNode(): if producingNode.killed: - relaunchNode(producingNode, producingNodeId) + relaunchNode(producingNode) # Give some time for it to produce, so lib is advancing waitForBlksProducedAndLibAdvanced() @@ -191,7 +191,7 @@ def executeTest(nodeIdOfNodeToTest, runTestScenario): try: # Relaunch killed node so it can be used for the test nodeToTest = cluster.getNode(nodeIdOfNodeToTest) - relaunchNode(nodeToTest, nodeIdOfNodeToTest, relaunchAssertMessage="Fail to relaunch before running test scenario") + relaunchNode(nodeToTest, relaunchAssertMessage="Fail to relaunch before running test scenario") # Run test scenario runTestScenario(nodeIdOfNodeToTest, nodeToTest) @@ -212,7 +212,7 @@ def replayInIrrModeWithRevBlks(nodeIdOfNodeToTest, nodeToTest): # Kill node and replay in irreversible mode nodeToTest.kill(signal.SIGTERM) - relaunchNode(nodeToTest, nodeIdOfNodeToTest, chainArg=" --read-mode irreversible --replay") + relaunchNode(nodeToTest, chainArg=" --read-mode irreversible --replay") # Confirm state confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest, headLibAndForkDbHeadBeforeSwitchMode) @@ -226,7 +226,7 @@ def replayInIrrModeWithoutRevBlks(nodeIdOfNodeToTest, nodeToTest): # Shut down node, remove reversible blks and relaunch nodeToTest.kill(signal.SIGTERM) removeReversibleBlks(nodeIdOfNodeToTest) - relaunchNode(nodeToTest, nodeIdOfNodeToTest, chainArg=" --read-mode irreversible --replay") + relaunchNode(nodeToTest, chainArg=" --read-mode irreversible --replay") # Ensure the node condition is as expected after relaunch confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest, headLibAndForkDbHeadBeforeSwitchMode) @@ -239,7 +239,7 @@ def switchSpecToIrrMode(nodeIdOfNodeToTest, nodeToTest): # Kill and relaunch in irreversible mode nodeToTest.kill(signal.SIGTERM) - relaunchNode(nodeToTest, nodeIdOfNodeToTest, chainArg=" --read-mode irreversible") + relaunchNode(nodeToTest, chainArg=" --read-mode irreversible") # Ensure the node condition is as expected after relaunch confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest, headLibAndForkDbHeadBeforeSwitchMode) @@ -252,7 +252,7 @@ def switchIrrToSpecMode(nodeIdOfNodeToTest, nodeToTest): # Kill and relaunch in speculative mode nodeToTest.kill(signal.SIGTERM) - relaunchNode(nodeToTest, nodeIdOfNodeToTest, addSwapFlags={"--read-mode": "speculative"}) + relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "speculative"}) # Ensure the node condition is as expected after relaunch confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest, headLibAndForkDbHeadBeforeSwitchMode) @@ -268,7 +268,7 @@ def switchSpecToIrrModeWithConnectedToProdNode(nodeIdOfNodeToTest, nodeToTest): # Kill and relaunch in irreversible mode nodeToTest.kill(signal.SIGTERM) waitForBlksProducedAndLibAdvanced() # Wait for some blks to be produced and lib advance - relaunchNode(nodeToTest, nodeIdOfNodeToTest, chainArg=" --read-mode irreversible") + relaunchNode(nodeToTest, chainArg=" --read-mode irreversible") # Ensure the node condition is as expected after relaunch ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest) @@ -287,7 +287,7 @@ def switchIrrToSpecModeWithConnectedToProdNode(nodeIdOfNodeToTest, nodeToTest): # Kill and relaunch in irreversible mode nodeToTest.kill(signal.SIGTERM) waitForBlksProducedAndLibAdvanced() # Wait for some blks to be produced and lib advance) - relaunchNode(nodeToTest, nodeIdOfNodeToTest, addSwapFlags={"--read-mode": "speculative"}) + relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "speculative"}) # Ensure the node condition is as expected after relaunch ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest) @@ -305,7 +305,7 @@ def replayInIrrModeWithRevBlksAndConnectedToProdNode(nodeIdOfNodeToTest, nodeToT # Kill node and replay in irreversible mode nodeToTest.kill(signal.SIGTERM) waitForBlksProducedAndLibAdvanced() # Wait - relaunchNode(nodeToTest, nodeIdOfNodeToTest, chainArg=" --read-mode irreversible --replay") + relaunchNode(nodeToTest, chainArg=" --read-mode irreversible --replay") # Ensure the node condition is as expected after relaunch ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest) @@ -325,7 +325,7 @@ def replayInIrrModeWithoutRevBlksAndConnectedToProdNode(nodeIdOfNodeToTest, node nodeToTest.kill(signal.SIGTERM) removeReversibleBlks(nodeIdOfNodeToTest) waitForBlksProducedAndLibAdvanced() # Wait - relaunchNode(nodeToTest, nodeIdOfNodeToTest, chainArg=" --read-mode irreversible --replay") + relaunchNode(nodeToTest, chainArg=" --read-mode irreversible --replay") # Ensure the node condition is as expected after relaunch ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest) @@ -353,7 +353,7 @@ def switchToSpecModeWithIrrModeSnapshot(nodeIdOfNodeToTest, nodeToTest): # Start from clean data dir, recover back up blocks, and then relaunch with irreversible snapshot removeState(nodeIdOfNodeToTest) recoverBackedupBlksDir(nodeIdOfNodeToTest) # this function will delete the existing blocks dir first - relaunchNode(nodeToTest, nodeIdOfNodeToTest, chainArg=" --snapshot {}".format(getLatestSnapshot(nodeIdOfNodeToTest)), addSwapFlags={"--read-mode": "speculative"}) + relaunchNode(nodeToTest, chainArg=" --snapshot {}".format(getLatestSnapshot(nodeIdOfNodeToTest)), addSwapFlags={"--read-mode": "speculative"}) confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest) # Ensure it automatically replays "reversible blocks", i.e. head lib and fork db should be the same headLibAndForkDbHeadAfterRelaunch = getHeadLibAndForkDbHead(nodeToTest) @@ -372,7 +372,7 @@ def switchToSpecModeWithIrrModeSnapshot(nodeIdOfNodeToTest, nodeToTest): # Relaunch the node again (using the same snapshot) # This time ensure it automatically replays both "irreversible blocks" and "reversible blocks", i.e. the end result should be the same as before shutdown removeState(nodeIdOfNodeToTest) - relaunchNode(nodeToTest, nodeIdOfNodeToTest) + relaunchNode(nodeToTest) headLibAndForkDbHeadAfterRelaunch = getHeadLibAndForkDbHead(nodeToTest) assert headLibAndForkDbHeadBeforeShutdown == headLibAndForkDbHeadAfterRelaunch, \ "Head, Lib, and Fork Db after relaunch is different {} vs {}".format(headLibAndForkDbHeadBeforeShutdown, headLibAndForkDbHeadAfterRelaunch) diff --git a/tests/nodeos_multiple_version_protocol_feature_test.py b/tests/nodeos_multiple_version_protocol_feature_test.py index e45ffe45d5..d08595ab52 100755 --- a/tests/nodeos_multiple_version_protocol_feature_test.py +++ b/tests/nodeos_multiple_version_protocol_feature_test.py @@ -36,10 +36,10 @@ cluster=Cluster(walletd=True) cluster.setWalletMgr(walletMgr) -def restartNode(node: Node, nodeId, chainArg=None, addSwapFlags=None, nodeosPath=None): +def restartNode(node: Node, chainArg=None, addSwapFlags=None, nodeosPath=None): if not node.killed: node.kill(signal.SIGTERM) - isRelaunchSuccess = node.relaunch(nodeId, chainArg, addSwapFlags=addSwapFlags, + isRelaunchSuccess = node.relaunch(chainArg, addSwapFlags=addSwapFlags, timeout=5, cachePopen=True, nodeosPath=nodeosPath) assert isRelaunchSuccess, "Fail to relaunch" @@ -61,7 +61,7 @@ def isDesiredProdTurn(): def waitForOneRound(): time.sleep(24) # We have 4 producers for this test -def setValidityOfActTimeSubjRestriction(node, nodeId, codename, valid): +def setValidityOfActTimeSubjRestriction(node, codename, valid): invalidActTimeSubjRestriction = { "earliest_allowed_activation_time": "2030-01-01T00:00:00.000", } @@ -69,8 +69,8 @@ def setValidityOfActTimeSubjRestriction(node, nodeId, codename, valid): "earliest_allowed_activation_time": "1970-01-01T00:00:00.000", } actTimeSubjRestriction = validActTimeSubjRestriction if valid else invalidActTimeSubjRestriction - node.modifyBuiltinPFSubjRestrictions(nodeId, codename, actTimeSubjRestriction) - restartNode(node, nodeId) + node.modifyBuiltinPFSubjRestrictions(codename, actTimeSubjRestriction) + restartNode(node) def waitUntilBlockBecomeIrr(node, blockNum, timeout=60): def hasBlockBecomeIrr(): @@ -142,8 +142,8 @@ def areNodesInSync(nodes:[Node]): # Therefore, 1st node will be out of sync with 2nd, 3rd, and 4th node # After a round has passed though, 1st node will realize he's in minority fork and then join the other nodes # Hence, the PREACTIVATE_FEATURE that was previously activated will be dropped and all of the nodes should be in sync - setValidityOfActTimeSubjRestriction(newNodes[1], newNodeIds[1], "PREACTIVATE_FEATURE", False) - setValidityOfActTimeSubjRestriction(newNodes[2], newNodeIds[2], "PREACTIVATE_FEATURE", False) + setValidityOfActTimeSubjRestriction(newNodes[1], "PREACTIVATE_FEATURE", False) + setValidityOfActTimeSubjRestriction(newNodes[2], "PREACTIVATE_FEATURE", False) waitUntilBeginningOfProdTurn(newNodes[0], "defproducera") newNodes[0].activatePreactivateFeature() @@ -163,8 +163,8 @@ def areNodesInSync(nodes:[Node]): # They will be in sync and their LIB will advance since they control > 2/3 of the producers # Also the LIB should be able to advance past the block that contains PREACTIVATE_FEATURE # However, the 4th node will be out of sync with them, and its LIB will stuck - setValidityOfActTimeSubjRestriction(newNodes[1], newNodeIds[1], "PREACTIVATE_FEATURE", True) - setValidityOfActTimeSubjRestriction(newNodes[2], newNodeIds[2], "PREACTIVATE_FEATURE", True) + setValidityOfActTimeSubjRestriction(newNodes[1], "PREACTIVATE_FEATURE", True) + setValidityOfActTimeSubjRestriction(newNodes[2], "PREACTIVATE_FEATURE", True) waitUntilBeginningOfProdTurn(newNodes[0], "defproducera") libBeforePreactivation = newNodes[0].getIrreversibleBlockNum() diff --git a/tests/nodeos_protocol_feature_test.py b/tests/nodeos_protocol_feature_test.py index df416da1c2..ef3cc3035a 100755 --- a/tests/nodeos_protocol_feature_test.py +++ b/tests/nodeos_protocol_feature_test.py @@ -30,10 +30,10 @@ # The following test case will test the Protocol Feature JSON reader of the blockchain -def restartNode(node: Node, nodeId, chainArg=None, addSwapFlags=None): +def restartNode(node: Node, chainArg=None, addSwapFlags=None): if not node.killed: node.kill(signal.SIGTERM) - isRelaunchSuccess = node.relaunch(nodeId, chainArg, addSwapFlags=addSwapFlags, timeout=5, cachePopen=True) + isRelaunchSuccess = node.relaunch(chainArg, addSwapFlags=addSwapFlags, timeout=5, cachePopen=True) assert isRelaunchSuccess, "Fail to relaunch" walletMgr=WalletMgr(True) @@ -56,8 +56,8 @@ def restartNode(node: Node, nodeId, chainArg=None, addSwapFlags=None): "preactivation_required": True, "enabled": False } - biosNode.modifyBuiltinPFSubjRestrictions("bios", "PREACTIVATE_FEATURE", newSubjectiveRestrictions) - restartNode(biosNode, "bios") + biosNode.modifyBuiltinPFSubjRestrictions("PREACTIVATE_FEATURE", newSubjectiveRestrictions) + restartNode(biosNode) supportedProtocolFeatureDict = biosNode.getSupportedProtocolFeatureDict() preactivateFeatureSubjectiveRestrictions = supportedProtocolFeatureDict["PREACTIVATE_FEATURE"]["subjective_restrictions"] diff --git a/tests/nodeos_startup_catchup.py b/tests/nodeos_startup_catchup.py index 47cfc2e7b3..f7676dd792 100755 --- a/tests/nodeos_startup_catchup.py +++ b/tests/nodeos_startup_catchup.py @@ -168,7 +168,7 @@ def waitForNodeStarted(node): catchupNode.interruptAndVerifyExitStatus(60) Print("Restart catchup node") - catchupNode.relaunch(catchupNodeNum, cachePopen=True) + catchupNode.relaunch(cachePopen=True) waitForNodeStarted(catchupNode) lastCatchupLibNum=lib(catchupNode) From 203db20e6ee982a1acd1922c234c570ad1f678ab Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 3 Jun 2020 12:19:29 -0500 Subject: [PATCH 02/12] Moved findFiles static method from Cluster to Node. --- tests/Cluster.py | 30 ++++++++---------------------- tests/Node.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index ca091f23c4..fbb5a177d2 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -1411,23 +1411,11 @@ def dumpErrorDetailImpl(fileName): else: Utils.Print("File %s not found." % (fileName)) - @staticmethod - def __findFiles(path): - files=[] - it=os.scandir(path) - for entry in it: - if entry.is_file(follow_symlinks=False): - match=re.match("stderr\..+\.txt", entry.name) - if match: - files.append(os.path.join(path, entry.name)) - files.sort() - return files - def dumpErrorDetails(self): fileName=Utils.getNodeConfigDir("bios", "config.ini") Cluster.dumpErrorDetailImpl(fileName) path=Utils.getNodeDataDir("bios") - fileNames=Cluster.__findFiles(path) + fileNames=Node.findFiles(path) for fileName in fileNames: Cluster.dumpErrorDetailImpl(fileName) @@ -1438,7 +1426,7 @@ def dumpErrorDetails(self): fileName=os.path.join(configLocation, "genesis.json") Cluster.dumpErrorDetailImpl(fileName) path=Utils.getNodeDataDir(i) - fileNames=Cluster.__findFiles(path) + fileNames=Node.findFiles(path) for fileName in fileNames: Cluster.dumpErrorDetailImpl(fileName) @@ -1558,14 +1546,12 @@ def getInfos(self, silentErrors=False, exitOnError=False): return infos def reportStatus(self): - if hasattr(self, "biosNode") and self.biosNode is not None: - self.biosNode.reportStatus() - if hasattr(self, "nodes"): - for node in self.nodes: - try: - node.reportStatus() - except: - Utils.Print("No reportStatus") + nodes = self.getAllNodes() + for node in nodes: + try: + node.reportStatus() + except: + Utils.Print("No reportStatus for nodeId: %s" % (node.nodeId)) def getBlockLog(self, nodeExtension, blockLogAction=BlockLogAction.return_blocks, outputFile=None, first=None, last=None, throwException=False, silentErrors=False, exitOnError=False): blockLogDir=Utils.getNodeDataDir(nodeExtension, "blocks") diff --git a/tests/Node.py b/tests/Node.py index 6152ad6aca..766c762425 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1326,3 +1326,15 @@ def modifyBuiltinPFSubjRestrictions(self, featureCodename, subjectiveRestriction def createSnapshot(self): param = { } return self.processCurlCmd("producer", "create_snapshot", json.dumps(param)) + + @staticmethod + def findFiles(path): + files=[] + it=os.scandir(path) + for entry in it: + if entry.is_file(follow_symlinks=False): + match=re.match("stderr\..+\.txt", entry.name) + if match: + files.append(os.path.join(path, entry.name)) + files.sort() + return files From 8bd7af16e0a54ecf492d37eb372b5d1d37bd8d31 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 3 Jun 2020 12:21:01 -0500 Subject: [PATCH 03/12] Added method to analyze production for a Node and added analysis to shutting down test. --- tests/Node.py | 57 +++++++++++++++++++++++++++++++++++++++++++++ tests/TestHelper.py | 14 ++++++++++- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/tests/Node.py b/tests/Node.py index 766c762425..d643c5ecdd 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1338,3 +1338,60 @@ def findFiles(path): files.append(os.path.join(path, entry.name)) files.sort() return files + + def analyzeProduction(self, specificBlockNum=None, thresholdMs=500): + dataDir=Utils.getNodeDataDir(self.nodeId) + files=Node.findFiles(dataDir) + blockAnalysis={} + anyBlockStr=r'[0-9]+' + initialTimestamp=r'\s+([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3})\s' + producedBlockPreStr=r'.+Produced\sblock\s+.+\s#(' + producedBlockPostStr=r')\s@\s([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3})' + anyBlockPtrn=re.compile(initialTimestamp + producedBlockPreStr + anyBlockStr + producedBlockPostStr) + producedBlockPtrn=re.compile(initialTimestamp + producedBlockPreStr + str(specificBlockNum) + producedBlockPostStr) if specificBlockNum is not None else anyBlockPtrn + producedBlockDonePtrn=re.compile(initialTimestamp + r'.+Producing\sBlock\s+#' + anyBlockStr + '\sreturned:\strue') + for file in files: + with open(file, 'r') as f: + line = f.readline() + while line: + readLine=True # assume we need to read the next line before the next pass + match = producedBlockPtrn.search(line) + if match: + prodTimeStr = match.group(1) + slotTimeStr = match.group(3) + blockNum = int(match.group(2)) + + line = f.readline() + while line: + matchNextBlock = anyBlockPtrn.search(line) + if matchNextBlock: + readLine=False #already have the next line ready to check on next pass + break + + matchBlockActuallyProduced = producedBlockDonePtrn.search(line) + if matchBlockActuallyProduced: + prodTimeStr = matchBlockActuallyProduced.group(1) + break + + line = f.readline() + + prodTime = datetime.strptime(prodTimeStr, Utils.TimeFmt) + slotTime = datetime.strptime(slotTimeStr, Utils.TimeFmt) + delta = prodTime - slotTime + millisecondsPerSecond=1000 + limit = timedelta(seconds=thresholdMs/millisecondsPerSecond) + if delta > limit: + if blockNum in blockAnalysis: + Utils.errorExit("Found repeat production of the same block num: %d in one of the stderr files in: %s" % (blockNum, dataDir)) + blockAnalysis[blockNum] = { "slot": slotTimeStr, "prod": prodTimeStr } + + if specificBlockNum is not None: + return blockAnalysis + elif specificBlockNum is not None: + blockAnalysis[blockNum] = { "slot": None, "prod": None} + return + + if readLine: + line = f.readline() + + return blockAnalysis diff --git a/tests/TestHelper.py b/tests/TestHelper.py index fcac66a870..bf4ff5cb2b 100644 --- a/tests/TestHelper.py +++ b/tests/TestHelper.py @@ -146,12 +146,24 @@ def shutdown(cluster, walletMgr, testSuccessful=True, killEosInstances=True, kil Utils.Print("Test succeeded.") else: Utils.Print("Test failed.") + + def reportProductionAnalysis(): + Utils.Print(Utils.FileDivider) + for node in cluster.getAllNodes(): + missedBlocks=node.analyzeProduction(thresholdMs=0) + if len(missedBlocks) > 0: + Utils.Print("NodeId: %s produced the following blocks late: %s" % (node.nodeId, missedBlocks)) + if not testSuccessful and dumpErrorDetails: cluster.reportStatus() Utils.Print(Utils.FileDivider) - psOut=Cluster.pgrepEosServers(timeout=60) + psOut = Cluster.pgrepEosServers(timeout=60) Utils.Print("pgrep output:\n%s" % (psOut)) + reportProductionAnalysis() Utils.Print("== Errors see above ==") + elif dumpErrorDetails: + # for now report these to know how many blocks we are missing production windows for + reportProductionAnalysis() if killEosInstances: Utils.Print("Shut down the cluster.") From d8bd4f0e8b09f2cf6d263cf15acca6537c150fea Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 3 Jun 2020 12:43:23 -0500 Subject: [PATCH 04/12] Added retrieving actual production time for determining how late blocks are processed. --- tests/ship_test.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/ship_test.py b/tests/ship_test.py index 8ea27a11e0..f7590b478a 100755 --- a/tests/ship_test.py +++ b/tests/ship_test.py @@ -169,15 +169,31 @@ blockNum = int(match.group(2)) if blockNum > maxFirstBN: # ship requests can only affect time after clients started - timeFmt = '%Y-%m-%dT%H:%M:%S.%f' - rcvTime = datetime.strptime(rcvTimeStr, timeFmt) - prodTime = datetime.strptime(prodTimeStr, timeFmt) + rcvTime = datetime.strptime(rcvTimeStr, Utils.TimeFmt) + prodTime = datetime.strptime(prodTimeStr, Utils.TimeFmt) delta = rcvTime - prodTime biggestDelta = max(delta, biggestDelta) totalDelta += delta timeCount += 1 - assert delta < timedelta(seconds=0.500), Print("ERROR: block_num: %s took %.3f seconds to be received." % (blockNum, delta.total_seconds())) + limit = timedelta(seconds=0.500) + if delta >= limit: + actualProducerTimeStr=None + nodes = cluster.getAllNodes() + nodes.remove(shipNodeNum) + for node in nodes: + threshold=-500 # set negative to guarantee the block analysis gets returned + blockAnalysis = node.analyzeProduction(specificBlockNum=blockNum, thresholdMs=threshold) + actualProducerTimeStr = blockAnalysis[blockNum]["prod"] + if actualProducerTimeStr is not None: + break + if actualProducerTimeStr is not None: + actualProducerTime = datetime.strptime(actualProducerTimeStr, Utils.TimeFmt) + if rcvTime - actualProducerTime >= limit: + actualProducerTime = None # let it fail below + + if actualProducerTimeStr is None: + Utils.errorExit("block_num: %s took %.3f seconds to be received." % (blockNum, delta.total_seconds())) line = f.readline() From ec813f2d4b7ad8b2779c81ed57566131ee1fca65 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 3 Jun 2020 13:58:44 -0500 Subject: [PATCH 05/12] Missed some methods that were still passing nodeId. --- tests/nodeos_multiple_version_protocol_feature_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/nodeos_multiple_version_protocol_feature_test.py b/tests/nodeos_multiple_version_protocol_feature_test.py index d08595ab52..cfee676fc4 100755 --- a/tests/nodeos_multiple_version_protocol_feature_test.py +++ b/tests/nodeos_multiple_version_protocol_feature_test.py @@ -191,11 +191,11 @@ def areNodesInSync(nodes:[Node]): portableRevBlkPath = os.path.join(Utils.getNodeDataDir(oldNodeId), "rev_blk_portable_format") oldNode.kill(signal.SIGTERM) # Note, for the following relaunch, these will fail to relaunch immediately (expected behavior of export/import), so the chainArg will not replace the old cmd - oldNode.relaunch(oldNodeId, chainArg="--export-reversible-blocks {}".format(portableRevBlkPath), timeout=1) - oldNode.relaunch(oldNodeId, chainArg="--import-reversible-blocks {}".format(portableRevBlkPath), timeout=1, nodeosPath="programs/nodeos/nodeos") + oldNode.relaunch(chainArg="--export-reversible-blocks {}".format(portableRevBlkPath), timeout=1) + oldNode.relaunch(chainArg="--import-reversible-blocks {}".format(portableRevBlkPath), timeout=1, nodeosPath="programs/nodeos/nodeos") os.remove(portableRevBlkPath) - restartNode(oldNode, oldNodeId, chainArg="--replay", nodeosPath="programs/nodeos/nodeos") + restartNode(oldNode, chainArg="--replay", nodeosPath="programs/nodeos/nodeos") time.sleep(2) # Give some time to replay assert areNodesInSync(allNodes), "All nodes should be in sync" From ee04c51d5a57c5eae04f8afd9347acc550676501 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 3 Jun 2020 14:28:28 -0500 Subject: [PATCH 06/12] Fixed removing of shipNode from node list. --- tests/ship_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/ship_test.py b/tests/ship_test.py index f7590b478a..8879550a10 100755 --- a/tests/ship_test.py +++ b/tests/ship_test.py @@ -179,8 +179,7 @@ limit = timedelta(seconds=0.500) if delta >= limit: actualProducerTimeStr=None - nodes = cluster.getAllNodes() - nodes.remove(shipNodeNum) + nodes = [node for node in cluster.getAllNodes() if node.nodeId != shipNodeNum] for node in nodes: threshold=-500 # set negative to guarantee the block analysis gets returned blockAnalysis = node.analyzeProduction(specificBlockNum=blockNum, thresholdMs=threshold) From 8da182dd27590f98c958ea7c75d9df2a911f6200 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 3 Jun 2020 14:47:21 -0500 Subject: [PATCH 07/12] Removing unneeded conversion. --- tests/Node.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index d643c5ecdd..7279c8b65c 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1378,8 +1378,7 @@ def analyzeProduction(self, specificBlockNum=None, thresholdMs=500): prodTime = datetime.strptime(prodTimeStr, Utils.TimeFmt) slotTime = datetime.strptime(slotTimeStr, Utils.TimeFmt) delta = prodTime - slotTime - millisecondsPerSecond=1000 - limit = timedelta(seconds=thresholdMs/millisecondsPerSecond) + limit = timedelta(milliseconds=thresholdMs) if delta > limit: if blockNum in blockAnalysis: Utils.errorExit("Found repeat production of the same block num: %d in one of the stderr files in: %s" % (blockNum, dataDir)) From 45104bca6dc7a6f0c5c114eee148b1827aa8f4c4 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 3 Jun 2020 14:57:14 -0500 Subject: [PATCH 08/12] Fixed PR comments. --- tests/Cluster.py | 11 ++++------- tests/Node.py | 4 ++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index fbb5a177d2..39e25c3d5b 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -62,7 +62,7 @@ def __init__(self, walletd=False, localCluster=True, host="localhost", port=8888 defproducerbPrvtKey: Defproducerb account private key """ self.accounts={} - self.nodes={} + self.nodes=[] self.unstartedNodes=[] self.localCluster=localCluster self.wallet=None @@ -672,10 +672,7 @@ def getNode(self, nodeId=0, exitOnError=True): return self.nodes[nodeId] def getNodes(self): - nodes = [] - if hasattr(self, "nodes") and self.nodes is not None: - nodes += self.nodes - return nodes + return self.nodes[:] def getAllNodes(self): nodes = [] @@ -1415,7 +1412,7 @@ def dumpErrorDetails(self): fileName=Utils.getNodeConfigDir("bios", "config.ini") Cluster.dumpErrorDetailImpl(fileName) path=Utils.getNodeDataDir("bios") - fileNames=Node.findFiles(path) + fileNames=Node.findStderrFiles(path) for fileName in fileNames: Cluster.dumpErrorDetailImpl(fileName) @@ -1426,7 +1423,7 @@ def dumpErrorDetails(self): fileName=os.path.join(configLocation, "genesis.json") Cluster.dumpErrorDetailImpl(fileName) path=Utils.getNodeDataDir(i) - fileNames=Node.findFiles(path) + fileNames=Node.findStderrFiles(path) for fileName in fileNames: Cluster.dumpErrorDetailImpl(fileName) diff --git a/tests/Node.py b/tests/Node.py index 7279c8b65c..55d606807a 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1328,7 +1328,7 @@ def createSnapshot(self): return self.processCurlCmd("producer", "create_snapshot", json.dumps(param)) @staticmethod - def findFiles(path): + def findStderrFiles(path): files=[] it=os.scandir(path) for entry in it: @@ -1341,7 +1341,7 @@ def findFiles(path): def analyzeProduction(self, specificBlockNum=None, thresholdMs=500): dataDir=Utils.getNodeDataDir(self.nodeId) - files=Node.findFiles(dataDir) + files=Node.findStderrFiles(dataDir) blockAnalysis={} anyBlockStr=r'[0-9]+' initialTimestamp=r'\s+([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3})\s' From e8b36e7db981eb5e0b7ad1466d95b6c9a7cffa6d Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 3 Jun 2020 17:50:58 -0500 Subject: [PATCH 09/12] Removed old code that wrote ship files to log before we added archiving. --- tests/ship_test.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/ship_test.py b/tests/ship_test.py index 8879550a10..8bcbf7b73f 100755 --- a/tests/ship_test.py +++ b/tests/ship_test.py @@ -203,26 +203,6 @@ finally: TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=killEosInstances, killWallet=killWallet, keepLogs=keepLogs, cleanRun=killAll, dumpErrorDetails=dumpErrorDetails) if shipTempDir is not None: - if dumpErrorDetails and not testSuccessful: - def printTruncatedFile(filename, maxLines): - Print(Utils.FileDivider) - with open(filename, "r") as f: - Print("Contents of %s" % (filename)) - line = f.readline() - lineCount = 0 - while line and lineCount < maxLines: - Print(line) - lineCount += 1 - line = f.readline() - if line: - Print("... CONTENT TRUNCATED AT %d lines" % (maxLines)) - - for index in range(0, args.num_clients): - # error file should not contain much content, so if there are lots of lines it is likely useless - printTruncatedFile("%s%d.err" % (shipClientFilePrefix, i), maxLines=1000) - # output file should have lots of output, but if user passes in a huge number of requests, these could go on forever - printTruncatedFile("%s%d.out" % (shipClientFilePrefix, i), maxLines=20000) - if testSuccessful and not keepLogs: shutil.rmtree(shipTempDir, ignore_errors=True) From d4db5bc5dbba2efd9b0e9da24576d10f4ef174c9 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 3 Jun 2020 18:05:54 -0500 Subject: [PATCH 10/12] Fixed using wrong block num variable and adjusted timing delta for reporting. --- tests/Node.py | 2 +- tests/TestHelper.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index 55d606807a..8df1e4fe06 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1387,7 +1387,7 @@ def analyzeProduction(self, specificBlockNum=None, thresholdMs=500): if specificBlockNum is not None: return blockAnalysis elif specificBlockNum is not None: - blockAnalysis[blockNum] = { "slot": None, "prod": None} + blockAnalysis[specificBlockNum] = { "slot": None, "prod": None} return if readLine: diff --git a/tests/TestHelper.py b/tests/TestHelper.py index bf4ff5cb2b..d07b362968 100644 --- a/tests/TestHelper.py +++ b/tests/TestHelper.py @@ -147,10 +147,10 @@ def shutdown(cluster, walletMgr, testSuccessful=True, killEosInstances=True, kil else: Utils.Print("Test failed.") - def reportProductionAnalysis(): + def reportProductionAnalysis(thresholdMs): Utils.Print(Utils.FileDivider) for node in cluster.getAllNodes(): - missedBlocks=node.analyzeProduction(thresholdMs=0) + missedBlocks=node.analyzeProduction(thresholdMs=thresholdMs) if len(missedBlocks) > 0: Utils.Print("NodeId: %s produced the following blocks late: %s" % (node.nodeId, missedBlocks)) @@ -159,11 +159,11 @@ def reportProductionAnalysis(): Utils.Print(Utils.FileDivider) psOut = Cluster.pgrepEosServers(timeout=60) Utils.Print("pgrep output:\n%s" % (psOut)) - reportProductionAnalysis() + reportProductionAnalysis(thresholdMs=0) Utils.Print("== Errors see above ==") elif dumpErrorDetails: # for now report these to know how many blocks we are missing production windows for - reportProductionAnalysis() + reportProductionAnalysis(thresholdMs=200) if killEosInstances: Utils.Print("Shut down the cluster.") From 5f6a573bd837a127becc47a68e15887d7635c6b7 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Thu, 4 Jun 2020 00:20:50 -0500 Subject: [PATCH 11/12] Fixed handling for when block is not produced in the first stderr file. --- tests/Node.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index 8df1e4fe06..6ef289e3aa 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1386,11 +1386,11 @@ def analyzeProduction(self, specificBlockNum=None, thresholdMs=500): if specificBlockNum is not None: return blockAnalysis - elif specificBlockNum is not None: - blockAnalysis[specificBlockNum] = { "slot": None, "prod": None} - return if readLine: line = f.readline() + if specificBlockNum is not None and specificBlockNum not in blockAnalysis: + blockAnalysis[specificBlockNum] = { "slot": None, "prod": None} + return blockAnalysis From 5d270f86535cf805eea0fb947ca0f3ffcd96b289 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Tue, 26 Apr 2022 22:27:11 -0400 Subject: [PATCH 12/12] left over nodeIndex parameter removal for Node.relaunch() calls --- tests/large-lib-test.py | 10 +++++----- tests/nodeos_irreversible_mode_test.py | 2 +- tests/nodeos_short_fork_take_over_test.py | 4 ++-- tests/nodeos_under_min_avail_ram.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/large-lib-test.py b/tests/large-lib-test.py index 0e53ccc5f0..bc9b872ffc 100755 --- a/tests/large-lib-test.py +++ b/tests/large-lib-test.py @@ -48,8 +48,8 @@ walletMgr=WalletMgr(True) cluster.setWalletMgr(walletMgr) -def relaunchNode(node: Node, nodeId, chainArg="", skipGenesis=True, relaunchAssertMessage="Fail to relaunch"): - isRelaunchSuccess=node.relaunch(nodeId, chainArg=chainArg, timeout=relaunchTimeout, skipGenesis=skipGenesis, cachePopen=True) +def relaunchNode(node: Node, chainArg="", skipGenesis=True, relaunchAssertMessage="Fail to relaunch"): + isRelaunchSuccess=node.relaunch(chainArg=chainArg, timeout=relaunchTimeout, skipGenesis=skipGenesis, cachePopen=True) time.sleep(1) # Give a second to replay or resync if needed assert isRelaunchSuccess, relaunchAssertMessage return isRelaunchSuccess @@ -103,9 +103,9 @@ def relaunchNode(node: Node, nodeId, chainArg="", skipGenesis=True, relaunchAsse Print ("Relaunch all cluster nodes instances.") # -e -p eosio for resuming production, skipGenesis=False for launch the same chain as before - relaunchNode(producingNode, 0, chainArg="-e -p eosio --sync-fetch-span 5 ", skipGenesis=False) - relaunchNode(speculativeNode1, 1, chainArg="--sync-fetch-span 5 ") - relaunchNode(speculativeNode2, 2, chainArg="--sync-fetch-span 5 ", skipGenesis=False) + relaunchNode(producingNode, chainArg="-e -p eosio --sync-fetch-span 5 ", skipGenesis=False) + relaunchNode(speculativeNode1, chainArg="--sync-fetch-span 5 ") + relaunchNode(speculativeNode2, chainArg="--sync-fetch-span 5 ", skipGenesis=False) Print("Note LIBs") prodLib = producingNode.getIrreversibleBlockNum() diff --git a/tests/nodeos_irreversible_mode_test.py b/tests/nodeos_irreversible_mode_test.py index e7caf14ec4..ca58d210ff 100755 --- a/tests/nodeos_irreversible_mode_test.py +++ b/tests/nodeos_irreversible_mode_test.py @@ -345,7 +345,7 @@ def switchToSpecModeWithIrrModeSnapshot(nodeIdOfNodeToTest, nodeToTest): backupBlksDir(nodeIdOfNodeToTest) # Relaunch in irreversible mode and create the snapshot - relaunchNode(nodeToTest, nodeIdOfNodeToTest, chainArg=" --read-mode irreversible") + relaunchNode(nodeToTest, chainArg=" --read-mode irreversible") confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest) nodeToTest.createSnapshot() nodeToTest.kill(signal.SIGTERM) diff --git a/tests/nodeos_short_fork_take_over_test.py b/tests/nodeos_short_fork_take_over_test.py index 22b1c1ec7c..bbf5f00022 100755 --- a/tests/nodeos_short_fork_take_over_test.py +++ b/tests/nodeos_short_fork_take_over_test.py @@ -339,12 +339,12 @@ def getMinHeadAndLib(prodNodes): Utils.errorExit("Expected the node 1 to have shutdown.") Print("Relaunching the non-producing bridge node to connect the node 0 (defproducera, defproducerb)") - if not nonProdNode.relaunch(nonProdNode.nodeNum, None): + if not nonProdNode.relaunch(None): errorExit("Failure - (non-production) node %d should have restarted" % (nonProdNode.nodeNum)) Print("Relaunch node 1 (defproducerc) and let it connect to brigde node that already synced up with node 0") time.sleep(10) - if not node1.relaunch(nodeId=1, chainArg=" --enable-stale-production "): + if not node1.relaunch(chainArg=" --enable-stale-production "): errorExit("Failure - (non-production) node 1 should have restarted") Print("Waiting to allow forks to resolve") diff --git a/tests/nodeos_under_min_avail_ram.py b/tests/nodeos_under_min_avail_ram.py index d48944344e..9a16017c79 100755 --- a/tests/nodeos_under_min_avail_ram.py +++ b/tests/nodeos_under_min_avail_ram.py @@ -231,7 +231,7 @@ def setName(self, num): if not enabledStaleProduction: addSwapFlags["--enable-stale-production"]="" # just enable stale production for the first node enabledStaleProduction=True - if not nodes[nodeIndex].relaunch(nodeIndex, "", newChain=False, addSwapFlags=addSwapFlags): + if not nodes[nodeIndex].relaunch("", newChain=False, addSwapFlags=addSwapFlags): Utils.cmdError("Failed to restart node0 with new capacity %s" % (maxRAMValue)) errorExit("Failure - Node should have restarted") addSwapFlags={} @@ -294,7 +294,7 @@ def setName(self, num): maxRAMValue=currentMinimumMaxRAM+5 currentMinimumMaxRAM=maxRAMValue addSwapFlags[maxRAMFlag]=str(maxRAMValue) - if not nodes[len(nodes)-1].relaunch(nodeIndex, "", newChain=False, addSwapFlags=addSwapFlags): + if not nodes[len(nodes)-1].relaunch("", newChain=False, addSwapFlags=addSwapFlags): Utils.cmdError("Failed to restart node %d with new capacity %s" % (numNodes-1, maxRAMValue)) errorExit("Failure - Node should have restarted") addSwapFlags={}