From a5e28370b36243f527ee67b87091d0a0b0df1cb9 Mon Sep 17 00:00:00 2001 From: "johnsonb@objectcomputing.com" Date: Tue, 10 Dec 2019 15:04:08 -0600 Subject: [PATCH 1/3] Fixed corner case for failures and added debug logging. --- tests/Node.py | 7 ++++++- tests/nodeos_under_min_avail_ram.py | 10 ++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index 3b9fa6e53ef..cd32c770852 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1363,7 +1363,9 @@ def interruptAndVerifyExitStatus(self, timeout=60): self.killed=True def verifyAlive(self, silent=False): - if not silent and Utils.Debug: Utils.Print("Checking if node(pid=%s) is alive(killed=%s): %s" % (self.pid, self.killed, self.cmd)) + logStatus=not silent and Utils.Debug + pid=self.pid + if logStatus: Utils.Print("Checking if node(pid=%s) is alive(killed=%s): %s" % (self.pid, self.killed, self.cmd)) if self.killed or self.pid is None: self.killed=True self.pid=None @@ -1375,10 +1377,13 @@ def verifyAlive(self, silent=False): # mark node as killed self.pid=None self.killed=True + if logStatus: Utils.Print("Determined node(formerly pid=%s) is killed" % (pid)) return False except PermissionError as ex: + if logStatus: Utils.Print("Determined node(formerly pid=%s) is killed" % (pid)) return True + if logStatus: Utils.Print("Determined node(pid=%s) is alive (kill command executed)" % (self.pid)) return True def getBlockProducerByNum(self, blockNum, timeout=None, waitForBlock=True, exitOnError=True): diff --git a/tests/nodeos_under_min_avail_ram.py b/tests/nodeos_under_min_avail_ram.py index 333e8136147..d2b5f4e0e3a 100755 --- a/tests/nodeos_under_min_avail_ram.py +++ b/tests/nodeos_under_min_avail_ram.py @@ -165,19 +165,19 @@ errorExit("Incorrect number of store actions sent") # Make sure all the nodes are shutdown (may take a little while for this to happen, so making multiple passes) - allDone=False count=0 - while not allDone: + while True: allDone=True for node in nodes: if node.verifyAlive(): allDone=False - if not allDone: - time.sleep(5) + if allDone: + break count+=1 if count>5: Utils.cmdError("All Nodes should have died") errorExit("Failure - All Nodes should have died") + time.sleep(5) for i in range(numNodes): f = open(Utils.getNodeDataDir(i) + "/stderr.txt") @@ -268,7 +268,6 @@ addSwapFlags={} time.sleep(10) - allDone=True for node in nodes: if not node.verifyAlive(): Utils.cmdError("All Nodes should be alive") @@ -305,7 +304,6 @@ time.sleep(10) Print("Check nodes are alive") - allDone=True for node in nodes: if not node.verifyAlive(): Utils.cmdError("All Nodes should be alive") From f1c9479ba9d0b48145b2eb8f263e48a5e5218445 Mon Sep 17 00:00:00 2001 From: "johnsonb@objectcomputing.com" Date: Tue, 10 Dec 2019 15:32:31 -0600 Subject: [PATCH 2/3] Increased time to wait for shutdown. --- tests/nodeos_under_min_avail_ram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nodeos_under_min_avail_ram.py b/tests/nodeos_under_min_avail_ram.py index d2b5f4e0e3a..57a47eaa83e 100755 --- a/tests/nodeos_under_min_avail_ram.py +++ b/tests/nodeos_under_min_avail_ram.py @@ -174,7 +174,7 @@ if allDone: break count+=1 - if count>5: + if count>12: Utils.cmdError("All Nodes should have died") errorExit("Failure - All Nodes should have died") time.sleep(5) From e6fadbe15a9e1d1d9c2417cb39ed2b4a867567da Mon Sep 17 00:00:00 2001 From: "johnsonb@objectcomputing.com" Date: Tue, 10 Dec 2019 19:53:54 -0600 Subject: [PATCH 3/3] Cleaned up log statements. --- tests/Node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Node.py b/tests/Node.py index cd32c770852..65160155075 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -1380,10 +1380,10 @@ def verifyAlive(self, silent=False): if logStatus: Utils.Print("Determined node(formerly pid=%s) is killed" % (pid)) return False except PermissionError as ex: - if logStatus: Utils.Print("Determined node(formerly pid=%s) is killed" % (pid)) + if logStatus: Utils.Print("Determined node(formerly pid=%s) is alive" % (pid)) return True - if logStatus: Utils.Print("Determined node(pid=%s) is alive (kill command executed)" % (self.pid)) + if logStatus: Utils.Print("Determined node(pid=%s) is alive" % (self.pid)) return True def getBlockProducerByNum(self, blockNum, timeout=None, waitForBlock=True, exitOnError=True):