From 707bca4a06378a903887ec765afc413c76ba7f61 Mon Sep 17 00:00:00 2001 From: Carlos Roig Date: Mon, 3 Dec 2018 16:46:57 +0100 Subject: [PATCH 1/3] Fixing return of incorrect subprocess --- kratos/python_scripts/run_tests.py | 99 ++++++++++++++++-------------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/kratos/python_scripts/run_tests.py b/kratos/python_scripts/run_tests.py index d87a9d668db4..cb885b67c5d4 100644 --- a/kratos/python_scripts/run_tests.py +++ b/kratos/python_scripts/run_tests.py @@ -1,16 +1,14 @@ from __future__ import print_function, absolute_import, division -import imp import os import re -import getopt import sys -import subprocess +import getopt import threading +import subprocess -from KratosMultiphysics import Tester -from KratosMultiphysics import KratosLoader -from KratosMultiphysics.KratosUnittest import SupressConsoleOutput, SupressConsoleError, SupressAllConsole +import KratosMultiphysics as KtsMp +import KratosMultiphysics.KratosUnittest as KtsUt def Usage(): @@ -42,7 +40,7 @@ def GetModulePath(module): ''' - return imp.find_module(module)[1] + return os.path.dirname(KtsMp.__file__) def GetAvailableApplication(): @@ -76,23 +74,21 @@ def __init__(self): self.exitCode = 0 def RunTestSuitInTime(self, application, applicationPath, path, level, verbose, command, timeout): - if(timeout > -1): - t = threading.Thread( - target=self.RunTestSuit, - args=(application, applicationPath, path, level, verbose, command) - ) + t = threading.Thread( + target=self.RunTestSuit, + args=(application, applicationPath, path, level, verbose, command) + ) - t.start() - t.join(timeout) + t.start() + t.join(timeout) - if t.isAlive(): - self.process.terminate() - t.join() - print('\nABORT: Tests for {} took to long. Process Killed.'.format(application), file=sys.stderr) - else: - print('\nTests for {} finished in time ({}s).'.format(application, timeout)) - else: - self.RunTestSuit(application, applicationPath, path, level, verbose, command) + if t.isAlive(): + self.process.terminate() + t.join() + print('\n[Error]: Tests for {} took to long. Process Killed.'.format(application), file=sys.stderr) + if self.exitCode != 0: + # If thread terminated with an error, propagate and terminate + sys.exit(self.exitCode) def RunTestSuit(self, application, applicationPath, path, level, verbose, command): ''' Calls the script that will run the tests. @@ -153,24 +149,33 @@ def RunTestSuit(self, application, applicationPath, path, level, verbose, comman file=sys.stderr) if os.path.isfile(script): - self.process = subprocess.Popen([ - command, - script, - '-l'+level, - '-v'+str(verbose) - ], stdout=subprocess.PIPE) - - # Used instead of wait to "soft-block" the process and prevent deadlocks - # and capture the first exit code different from OK - process_stdout, process_stderr = self.process.communicate() - if process_stdout: - print(process_stdout.decode('ascii'), file=sys.stdout) - if process_stderr: - print(process_stderr.decode('ascii'), file=sys.stderr) - - # Running out of time in the tests will send the error code -15. We may want to skip - # that one in a future. Right now will throw everything different from 0. - self.exitCode = int(self.process.returncode != 0) + try: + self.process = subprocess.Popen([ + command, + script, + '-l'+level, + '-v'+str(verbose) + ], stdout=subprocess.PIPE) + except OSError: + # Command does not exist + print('[Error]: Unable to execute {}'.format(command), file=sys.stderr) + self.exitCode = 1 + except ValueError: + # Command does exist, but the arguments are invalid (It sohuld never enter here. Just to be safe) + print('[Error]: Invalid arguments when calling {} {} {} {}'.format(command, script, '-l'+level, '-v'+str(verbose)), file=sys.stderr) + self.exitCode = 1 + else: + # Used instead of wait to "soft-block" the process and prevent deadlocks + # and capture the first exit code different from OK + process_stdout, process_stderr = self.process.communicate() + if process_stdout: + print(process_stdout.decode('ascii'), file=sys.stdout) + if process_stderr: + print(process_stderr.decode('ascii'), file=sys.stderr) + + # Running out of time in the tests will send the error code -15. We may want to skip + # that one in a future. Right now will throw everything different from 0. + self.exitCode = int(self.process.returncode != 0) else: if verbose > 0: print( @@ -196,8 +201,8 @@ def RunCppTests(self, applications): __import__("KratosMultiphysics." + application) try: - Tester.SetVerbosity(Tester.Verbosity.PROGRESS) - self.exitCode = Tester.RunAllTestCases() + KtsMp.Tester.SetVerbosity(KtsMp.Tester.Verbosity.PROGRESS) + self.exitCode = KtsMp.Tester.RunAllTestCases() except Exception as e: print('[Warning]:', e, file=sys.stderr) self.exitCode = 1 @@ -283,7 +288,7 @@ def main(): assert False, 'unhandled option' # Set timeout of the different levels - signalTime = int(-1) + signalTime = None if level == 'small': signalTime = int(60) elif level == 'nightly': @@ -295,7 +300,7 @@ def main(): # KratosCore must always be runned print('Running tests for KratosCore', file=sys.stderr) - with SupressConsoleOutput(): + with KtsUt.SupressConsoleOutput(): commander.RunTestSuitInTime( 'KratosCore', 'kratos', @@ -313,11 +318,11 @@ def main(): print('Running tests for {}'.format(application), file=sys.stderr) sys.stderr.flush() - with SupressConsoleOutput(): + with KtsUt.SupressConsoleOutput(): commander.RunTestSuitInTime( application, application, - KratosLoader.kratos_applications+'/', + KtsMp.KratosLoader.kratos_applications+'/', level, verbosity, cmd, @@ -328,7 +333,7 @@ def main(): # Run the cpp tests (does the same as run_cpp_tests.py) print('Running cpp tests', file=sys.stderr) - with SupressConsoleOutput(): + with KtsUt.SupressConsoleOutput(): commander.RunCppTests(applications) exit_code = max(exit_code, commander.exitCode) From 9a1820b369cd8b10a8f89d55b6a8e190adc5d5e0 Mon Sep 17 00:00:00 2001 From: philbucher Date: Fri, 4 Jan 2019 09:37:21 +0100 Subject: [PATCH 2/3] updated to new functions --- kratos/python_scripts/run_tests.py | 41 +++--------------------------- 1 file changed, 4 insertions(+), 37 deletions(-) diff --git a/kratos/python_scripts/run_tests.py b/kratos/python_scripts/run_tests.py index f537925f8f32..611f9ca482c6 100644 --- a/kratos/python_scripts/run_tests.py +++ b/kratos/python_scripts/run_tests.py @@ -9,6 +9,7 @@ import KratosMultiphysics as KtsMp import KratosMultiphysics.KratosUnittest as KtsUt +import KratosMultiphysics.kratos_utilities as KtsUtls def Usage(): @@ -30,40 +31,6 @@ def Usage(): print(l) -def GetModulePath(module): - ''' Returns the location of a module using its absolute path - - Return - ------ - string - The absolute path of the module - - ''' - - return os.path.dirname(KtsMp.__file__) - - -def GetAvailableApplication(): - ''' Return the list of applications available in KratosMultiphysics - - Return a list of compiled applications available in the KratosMultiphysics - module. - - Return - ------ - list of string - List of the names of the applications - - ''' - kratosPath = GetModulePath('KratosMultiphysics') - - apps = [ - f.split('.')[0] for f in os.listdir(kratosPath) if re.match(r'.*Application*', f) - ] - - return apps - - def handler(signum, frame): raise Exception("End of time") @@ -211,13 +178,13 @@ def RunCppTests(self, applications): def main(): # Define the command - cmd = os.path.dirname(GetModulePath('KratosMultiphysics'))+'/'+'runkratos' + cmd = os.path.join(os.path.dirname(KtsUtls.GetKratosMultiphysicsPath()), 'runkratos') verbose_values = [0, 1, 2] level_values = ['all', 'nightly', 'small', 'validation', 'mpi_all', 'mpi_small', 'mpi_nightly', 'mpi_validation'] # Set default values - applications = GetAvailableApplication() + applications = KtsUtls.GetListOfAvailableApplications() verbosity = 1 level = 'all' @@ -304,7 +271,7 @@ def main(): commander.RunTestSuitInTime( 'KratosCore', 'kratos', - os.path.dirname(GetModulePath('KratosMultiphysics')), + os.path.dirname(KtsUtls.GetKratosMultiphysicsPath()), level, verbosity, cmd, From 1b4dc23b245a65582b7750c29330158efcbadf50 Mon Sep 17 00:00:00 2001 From: Carlos Roig Date: Tue, 29 Jan 2019 14:52:26 +0100 Subject: [PATCH 3/3] Removing preemtive exit --- kratos/python_scripts/run_tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/kratos/python_scripts/run_tests.py b/kratos/python_scripts/run_tests.py index cb885b67c5d4..bb6a0273118b 100644 --- a/kratos/python_scripts/run_tests.py +++ b/kratos/python_scripts/run_tests.py @@ -86,9 +86,6 @@ def RunTestSuitInTime(self, application, applicationPath, path, level, verbose, self.process.terminate() t.join() print('\n[Error]: Tests for {} took to long. Process Killed.'.format(application), file=sys.stderr) - if self.exitCode != 0: - # If thread terminated with an error, propagate and terminate - sys.exit(self.exitCode) def RunTestSuit(self, application, applicationPath, path, level, verbose, command): ''' Calls the script that will run the tests.