From add9cbd1374e5ca376e72bfd1f903b43b9ea5a98 Mon Sep 17 00:00:00 2001 From: jimboid Date: Fri, 9 Dec 2016 11:35:51 +0000 Subject: [PATCH 01/22] Fix for bad initialiser. Reinitialising a parameter inside a for loop should not come as a suprise when things go missing. This has been fixed and the tests modified to take account. --- Longbow/corelibs/applications.py | 19 +++++++++--------- .../test_proccommandline.py | 10 +++++----- .../corelibs_applications/test_processjobs.py | 4 ++-- .../corelibs_applications/test_procfiles.py | 20 +++++++++---------- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/Longbow/corelibs/applications.py b/Longbow/corelibs/applications.py index 155cf6b..b392d93 100755 --- a/Longbow/corelibs/applications.py +++ b/Longbow/corelibs/applications.py @@ -135,6 +135,7 @@ def processjobs(jobs): filelist = [] appplugins = getattr(apps, "PLUGINEXECS") app = appplugins[jobs[job]["executable"]] + foundflags = [] substitution = {} LOG.debug("Command-line arguments for job '%s' are '%s'", @@ -178,7 +179,8 @@ def processjobs(jobs): pass # Process the command-line. - foundflags = _proccommandline(jobs[job], filelist, substitution) + foundflags = _proccommandline(jobs[job], filelist, foundflags, + substitution) # Validate if all required flags are present. _flagvalidator(jobs[job], foundflags) @@ -237,7 +239,7 @@ def _flagvalidator(job, foundflags): .format(job["jobname"], flags, app)) -def _proccommandline(job, filelist, substitution): +def _proccommandline(job, filelist, foundflags, substitution): """Command-line processor. This method selects which type of command-line we have. @@ -245,7 +247,6 @@ def _proccommandline(job, filelist, substitution): """ # Initialisation. args = list(job["executableargs"]) - initargs = list(job["executableargs"]) try: @@ -254,13 +255,13 @@ def _proccommandline(job, filelist, substitution): if args[0] == "<" and len(args) > 1: # Command-line type exec < input.file - foundflags = _procfiles(job, args[1], initargs, filelist, + foundflags = _procfiles(job, args[1], filelist, foundflags, substitution) elif len(args) == 1 and args[0] != "<": # Command-line type exec input.file - foundflags = _procfiles(job, args[0], initargs, filelist, + foundflags = _procfiles(job, args[0], filelist, foundflags, substitution) elif "-" in args[0]: @@ -268,7 +269,7 @@ def _proccommandline(job, filelist, substitution): for arg in args: # Command-line type exec -i file -c file - foundflags = _procfiles(job, arg, initargs, filelist, + foundflags = _procfiles(job, arg, filelist, foundflags, substitution) elif "-" not in args[0] and "-" in args[1]: @@ -276,7 +277,7 @@ def _proccommandline(job, filelist, substitution): for arg in args[1:]: # Command-line type exec subexec -i file -c file - foundflags = _procfiles(job, arg, initargs, filelist, + foundflags = _procfiles(job, arg, filelist, foundflags, substitution) else: @@ -294,12 +295,12 @@ def _proccommandline(job, filelist, substitution): return foundflags -def _procfiles(job, arg, initargs, filelist, substitution): +def _procfiles(job, arg, filelist, foundflags, substitution): """Processor for finding flags and files.""" # Initialisation. appplugins = getattr(apps, "PLUGINEXECS") app = appplugins[job["executable"]] - foundflags = [] + initargs = list(job["executableargs"]) # Check for as many files as there are replicates (default of 1). for rep in range(1, int(job["replicates"]) + 1): diff --git a/Tests/unit/corelibs_applications/test_proccommandline.py b/Tests/unit/corelibs_applications/test_proccommandline.py index a72050b..1ab0666 100644 --- a/Tests/unit/corelibs_applications/test_proccommandline.py +++ b/Tests/unit/corelibs_applications/test_proccommandline.py @@ -48,7 +48,7 @@ def test_proccommandline_test1(m_procfiles): "executableargs": ["<", "input.file"] } - apps._proccommandline(job, [], {}) + apps._proccommandline(job, [], [], {}) assert m_procfiles.call_count == 1 assert m_procfiles.call_args[0][1] == "input.file" @@ -66,7 +66,7 @@ def test_proccommandline_test2(m_procfiles): "executableargs": ["-c", "file", "-i", "file", "-p", "file"] } - apps._proccommandline(job, [], {}) + apps._proccommandline(job, [], [], {}) assert m_procfiles.call_count == 6 assert m_procfiles.call_args[0][1] == "file" @@ -84,7 +84,7 @@ def test_proccommandline_test3(m_procfiles): "executableargs": ["mdrun_mpi", "-deffnm", "filename"] } - apps._proccommandline(job, [], {}) + apps._proccommandline(job, [], [], {}) assert m_procfiles.call_count == 2 assert m_procfiles.call_args[0][1] == "filename" @@ -102,7 +102,7 @@ def test_proccommandline_test4(m_procfiles): "executableargs": ["input.file"] } - apps._proccommandline(job, [], {}) + apps._proccommandline(job, [], [], {}) assert m_procfiles.call_count == 1 assert m_procfiles.call_args[0][1] == "input.file" @@ -122,4 +122,4 @@ def test_proccommandline_except(): with pytest.raises(exceptions.RequiredinputError): - apps._proccommandline(job, [], {}) + apps._proccommandline(job, [], [], {}) diff --git a/Tests/unit/corelibs_applications/test_processjobs.py b/Tests/unit/corelibs_applications/test_processjobs.py index 1b83aa0..30ac402 100644 --- a/Tests/unit/corelibs_applications/test_processjobs.py +++ b/Tests/unit/corelibs_applications/test_processjobs.py @@ -38,7 +38,7 @@ import Longbow.corelibs.exceptions as exceptions -def _proccommandline(job, filelist, _): +def _proccommandline(job, filelist, foundfile, _): """Quick method to mock functionality""" for index, arg in enumerate(job["executableargs"]): @@ -47,7 +47,7 @@ def _proccommandline(job, filelist, _): filelist.append(job["executableargs"][index + 1]) - return [] + return foundfile def test_processjobs_abspath(): diff --git a/Tests/unit/corelibs_applications/test_procfiles.py b/Tests/unit/corelibs_applications/test_procfiles.py index 780671f..a7e127a 100644 --- a/Tests/unit/corelibs_applications/test_procfiles.py +++ b/Tests/unit/corelibs_applications/test_procfiles.py @@ -34,16 +34,16 @@ def test_procfiles_amber(): arg = "coords" filelist = [] + foundflags = [] job = { "executable": "pmemd.MPI", "replicates": "1", "localworkdir": "Tests/standards/jobs/single", "executableargs": ["-i", "input", "-c", "coords", "-p", "topol"] } - initargs = ["-i", "input", "-c", "coords", "-p", "topol"] substitution = {} - foundflags = apps._procfiles(job, arg, initargs, filelist, substitution) + foundflags = apps._procfiles(job, arg, filelist, foundflags, substitution) assert foundflags == ["-c"] assert filelist == ["coords"] @@ -58,16 +58,16 @@ def test_procfiles_charmm(): arg = "topol" filelist = [] + foundflags = [] job = { "executable": "charmm", "replicates": "1", "localworkdir": "Tests/standards/jobs/single", "executableargs": ["<", "topol"] } - initargs = ["<", "topol"] substitution = {} - foundflags = apps._procfiles(job, arg, initargs, filelist, substitution) + foundflags = apps._procfiles(job, arg, filelist, foundflags, substitution) assert foundflags == ["<"] assert filelist == ["topol"] @@ -82,16 +82,16 @@ def test_procfiles_gromacs(): arg = "test" filelist = [] + foundflags = [] job = { "executable": "mdrun_mpi", "replicates": "1", "localworkdir": "Tests/standards/jobs/single", "executableargs": ["-deffnm", "test"] } - initargs = ["-deffnm", "test"] substitution = {} - foundflags = apps._procfiles(job, arg, initargs, filelist, substitution) + foundflags = apps._procfiles(job, arg, filelist, foundflags, substitution) assert foundflags == ["-deffnm"] assert filelist == ["test.tpr"] @@ -106,16 +106,16 @@ def test_procfiles_namd(): arg = "input" filelist = [] + foundflags = [] job = { "executable": "namd2", "replicates": "1", "localworkdir": "Tests/standards/jobs/single", "executableargs": ["input"] } - initargs = ["input"] substitution = {} - foundflags = apps._procfiles(job, arg, initargs, filelist, substitution) + foundflags = apps._procfiles(job, arg, filelist, foundflags, substitution) assert foundflags == ["<"] assert filelist == ["input"] @@ -129,16 +129,16 @@ def test_procfiles_reps(): arg = "coords" filelist = [] + foundflags = [] job = { "executable": "pmemd.MPI", "replicates": "3", "localworkdir": "Tests/standards/jobs/replicate", "executableargs": ["-i", "input", "-c", "coords", "-p", "topol"] } - initargs = ["-i", "input", "-c", "coords", "-p", "topol"] substitution = {} - foundflags = apps._procfiles(job, arg, initargs, filelist, substitution) + foundflags = apps._procfiles(job, arg, filelist, foundflags, substitution) assert foundflags == ["-c"] assert filelist == ["rep1", "rep1/coords", "rep2", "rep2/coords", "rep3", From 2304fe6d2d0a282004917c57dd37b64517a1c2c0 Mon Sep 17 00:00:00 2001 From: jimboid Date: Fri, 9 Dec 2016 11:38:22 +0000 Subject: [PATCH 02/22] Nonsensical log message A message that had poor grammar and spacing issues has now been fixed. --- Longbow/corelibs/scheduling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Longbow/corelibs/scheduling.py b/Longbow/corelibs/scheduling.py index 7b9b516..c3470ba 100755 --- a/Longbow/corelibs/scheduling.py +++ b/Longbow/corelibs/scheduling.py @@ -199,7 +199,7 @@ def monitor(jobs): structure. """ - LOG.info("Monitoring job/s, depending on the chosen logging mode Longbow" + LOG.info("Monitoring job/s. Depending on the chosen logging mode, Longbow " "might appear to be doing nothing. Please be patient!") stageinterval, pollinterval = _monitorinitialise(jobs) From c9ab9cc2cf362eb988a2eecd1ef3c514f6243904 Mon Sep 17 00:00:00 2001 From: jimboid Date: Fri, 9 Dec 2016 13:52:56 +0000 Subject: [PATCH 03/22] Fixes for: Jobs being marked as complete prematurely due to zero length strings throwing an exception on checks in status() disabling application checks at the top level rather than within module. --- Longbow/corelibs/applications.py | 4 +--- Longbow/corelibs/configuration.py | 1 - Longbow/corelibs/entrypoints.py | 4 +++- Longbow/corelibs/scheduling.py | 2 +- Longbow/schedulers/lsf.py | 6 +++++- Longbow/schedulers/pbs.py | 6 +++++- Longbow/schedulers/sge.py | 6 +++++- Longbow/schedulers/slurm.py | 6 +++++- Longbow/schedulers/soge.py | 6 +++++- 9 files changed, 30 insertions(+), 11 deletions(-) diff --git a/Longbow/corelibs/applications.py b/Longbow/corelibs/applications.py index b392d93..6cc5f36 100755 --- a/Longbow/corelibs/applications.py +++ b/Longbow/corelibs/applications.py @@ -74,9 +74,7 @@ def testapp(jobs): checked[jobs[job]["resource"]] = [] # Now check if we have tested this exec already. - if (jobs[job]["executable"] not in checked[jobs[job]["resource"]] and - (jobs[job]["nochecks"] is False or - jobs[job]["nochecks"] == "false")): + if (jobs[job]["executable"] not in checked[jobs[job]["resource"]]): # If not then add it to the list now. checked[jobs[job]["resource"]].extend([jobs[job]["executable"]]) diff --git a/Longbow/corelibs/configuration.py b/Longbow/corelibs/configuration.py index 88d8c81..b26f0d8 100755 --- a/Longbow/corelibs/configuration.py +++ b/Longbow/corelibs/configuration.py @@ -83,7 +83,6 @@ "modules": "", "maxtime": "24:00", "memory": "", - "nochecks": False, "scripts": "", "staging-frequency": "300", "sge-peflag": "mpi", diff --git a/Longbow/corelibs/entrypoints.py b/Longbow/corelibs/entrypoints.py index 5ffbf44..71eacad 100644 --- a/Longbow/corelibs/entrypoints.py +++ b/Longbow/corelibs/entrypoints.py @@ -262,7 +262,9 @@ def longbowmain(parameters): # Test that for the applications listed in the job configuration # file are available and that the executable is present. - applications.testapp(jobs) + if parameters["nochecks"] is True: + + applications.testapp(jobs) # Process the jobs command line arguments and find files for # staging. diff --git a/Longbow/corelibs/scheduling.py b/Longbow/corelibs/scheduling.py index c3470ba..563f57f 100755 --- a/Longbow/corelibs/scheduling.py +++ b/Longbow/corelibs/scheduling.py @@ -503,7 +503,7 @@ def _monitorinitialise(jobs): } # This should always be present. - if "laststatus" not in job: + if "laststatus" not in jobs[job]: jobs[job]["laststatus"] = "" diff --git a/Longbow/schedulers/lsf.py b/Longbow/schedulers/lsf.py index 0496da7..282a0c7 100755 --- a/Longbow/schedulers/lsf.py +++ b/Longbow/schedulers/lsf.py @@ -198,7 +198,7 @@ def status(job): line = line.split() - if job["jobid"] in line[0]: + if len(line) > 0 and job["jobid"] in line[0]: jobstate = states[line[2]] break @@ -207,6 +207,10 @@ def status(job): jobstate = "Finished" + if jobstate == "": + + jobstate = "Finished" + return jobstate diff --git a/Longbow/schedulers/pbs.py b/Longbow/schedulers/pbs.py index a96a8e5..0f91334 100755 --- a/Longbow/schedulers/pbs.py +++ b/Longbow/schedulers/pbs.py @@ -243,7 +243,7 @@ def status(job): line = line.split() - if job["jobid"] in line[0]: + if len(line) > 0 and job["jobid"] in line[0]: jobstate = states[line[9]] break @@ -252,6 +252,10 @@ def status(job): jobstate = "Finished" + if jobstate == "": + + jobstate = "Finished" + return jobstate diff --git a/Longbow/schedulers/sge.py b/Longbow/schedulers/sge.py index e69a3f2..53e86b5 100755 --- a/Longbow/schedulers/sge.py +++ b/Longbow/schedulers/sge.py @@ -189,7 +189,7 @@ def status(job): line = line.split() - if job["jobid"] in line[0]: + if len(line) > 0 and job["jobid"] in line[0]: jobstate = states[line[4]] break @@ -198,6 +198,10 @@ def status(job): jobstate = "Finished" + if jobstate == "": + + jobstate = "Finished" + return jobstate diff --git a/Longbow/schedulers/slurm.py b/Longbow/schedulers/slurm.py index 88b92dd..9503187 100755 --- a/Longbow/schedulers/slurm.py +++ b/Longbow/schedulers/slurm.py @@ -204,7 +204,7 @@ def status(job): line = line.split() - if job["jobid"] in line[0]: + if len(line) > 0 and job["jobid"] in line[0]: jobstate = states[line[4]] break @@ -213,6 +213,10 @@ def status(job): jobstate = "Finished" + if jobstate == "": + + jobstate = "Finished" + return jobstate diff --git a/Longbow/schedulers/soge.py b/Longbow/schedulers/soge.py index ea2acdc..e9af283 100755 --- a/Longbow/schedulers/soge.py +++ b/Longbow/schedulers/soge.py @@ -206,7 +206,7 @@ def status(job): line = line.split() - if job["jobid"] in line[0]: + if len(line) > 0 and job["jobid"] in line[0]: jobstate = states[line[4]] break @@ -215,6 +215,10 @@ def status(job): jobstate = "Finished" + if jobstate == "": + + jobstate = "Finished" + return jobstate From b6499947aa9760fa2db8ecd3cd5eef9a7a22b4a0 Mon Sep 17 00:00:00 2001 From: jimboid Date: Fri, 9 Dec 2016 13:53:30 +0000 Subject: [PATCH 04/22] bad indentation copy & paste mistake --- Longbow/schedulers/slurm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Longbow/schedulers/slurm.py b/Longbow/schedulers/slurm.py index 9503187..6ea4b45 100755 --- a/Longbow/schedulers/slurm.py +++ b/Longbow/schedulers/slurm.py @@ -213,7 +213,7 @@ def status(job): jobstate = "Finished" - if jobstate == "": + if jobstate == "": jobstate = "Finished" From f4d10fc35ba6742ee637215de2a9704cc0920c51 Mon Sep 17 00:00:00 2001 From: jimboid Date: Fri, 9 Dec 2016 14:01:08 +0000 Subject: [PATCH 05/22] Fix the tests --- Longbow/corelibs/entrypoints.py | 2 +- .../corelibs_entrypoints/test_longbowmain.py | 56 ++++++++++++++++--- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/Longbow/corelibs/entrypoints.py b/Longbow/corelibs/entrypoints.py index 71eacad..9dedcc7 100644 --- a/Longbow/corelibs/entrypoints.py +++ b/Longbow/corelibs/entrypoints.py @@ -262,7 +262,7 @@ def longbowmain(parameters): # Test that for the applications listed in the job configuration # file are available and that the executable is present. - if parameters["nochecks"] is True: + if parameters["nochecks"] is False: applications.testapp(jobs) diff --git a/Tests/unit/corelibs_entrypoints/test_longbowmain.py b/Tests/unit/corelibs_entrypoints/test_longbowmain.py index dcaf525..246eae4 100644 --- a/Tests/unit/corelibs_entrypoints/test_longbowmain.py +++ b/Tests/unit/corelibs_entrypoints/test_longbowmain.py @@ -53,7 +53,8 @@ def test_longbowmain_disconnect(m_procconf, m_testcon, m_testenv, m_testapp, params = { "hosts": "some/file", - "disconnect": True + "disconnect": True, + "nochecks": False } mains.longbowmain(params) @@ -79,9 +80,9 @@ def test_longbowmain_disconnect(m_procconf, m_testcon, m_testenv, m_testapp, @mock.patch('Longbow.corelibs.scheduling.testenv') @mock.patch('Longbow.corelibs.shellwrappers.testconnections') @mock.patch('Longbow.corelibs.configuration.processconfigs') -def test_longbowmain_testcalls(m_procconf, m_testcon, m_testenv, m_testapp, - m_procjob, m_schedprep, m_stagup, m_sub, m_mon, - m_clean): +def test_longbowmain_testcalls1(m_procconf, m_testcon, m_testenv, m_testapp, + m_procjob, m_schedprep, m_stagup, m_sub, m_mon, + m_clean): """ Check that the correct function calls are made. @@ -89,7 +90,8 @@ def test_longbowmain_testcalls(m_procconf, m_testcon, m_testenv, m_testapp, params = { "hosts": "some/file", - "disconnect": False + "disconnect": False, + "nochecks": False } mains.longbowmain(params) @@ -106,6 +108,44 @@ def test_longbowmain_testcalls(m_procconf, m_testcon, m_testenv, m_testapp, assert m_clean.call_count == 1 +@mock.patch('Longbow.corelibs.staging.cleanup') +@mock.patch('Longbow.corelibs.scheduling.monitor') +@mock.patch('Longbow.corelibs.scheduling.submit') +@mock.patch('Longbow.corelibs.staging.stage_upstream') +@mock.patch('Longbow.corelibs.scheduling.prepare') +@mock.patch('Longbow.corelibs.applications.processjobs') +@mock.patch('Longbow.corelibs.applications.testapp') +@mock.patch('Longbow.corelibs.scheduling.testenv') +@mock.patch('Longbow.corelibs.shellwrappers.testconnections') +@mock.patch('Longbow.corelibs.configuration.processconfigs') +def test_longbowmain_testcalls2(m_procconf, m_testcon, m_testenv, m_testapp, + m_procjob, m_schedprep, m_stagup, m_sub, m_mon, + m_clean): + + """ + Check that the correct function calls are made. + """ + + params = { + "hosts": "some/file", + "disconnect": False, + "nochecks": True + } + + mains.longbowmain(params) + + assert m_procconf.call_count == 1 + assert m_testcon.call_count == 1 + assert m_testenv.call_count == 1 + assert m_testapp.call_count == 0 + assert m_procjob.call_count == 1 + assert m_schedprep.call_count == 1 + assert m_stagup.call_count == 1 + assert m_sub.call_count == 1 + assert m_mon.call_count == 1 + assert m_clean.call_count == 1 + + @mock.patch('Longbow.corelibs.staging.cleanup') @mock.patch('Longbow.corelibs.staging.stage_downstream') @mock.patch('Longbow.corelibs.scheduling.delete') @@ -128,7 +168,8 @@ def test_longbowmain_killrunning(m_procconf, m_testcon, m_testenv, m_testapp, params = { "hosts": "some/file", - "disconnect": False + "disconnect": False, + "nochecks": False } m_procconf.return_value = { @@ -180,7 +221,8 @@ def test_longbowmain_killcomplete(m_procconf, m_testcon, m_testenv, m_testapp, params = { "hosts": "some/file", - "disconnect": False + "disconnect": False, + "nochecks": False } m_procconf.return_value = { From a397aba03046c188948e7f55bba054ddf11be0ce Mon Sep 17 00:00:00 2001 From: jimboid Date: Fri, 9 Dec 2016 14:05:35 +0000 Subject: [PATCH 06/22] Removed tests. Removed the tests for nochecks in apps. These are not needed anymore. --- .../corelibs_applications/test_testapp.py | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/Tests/unit/corelibs_applications/test_testapp.py b/Tests/unit/corelibs_applications/test_testapp.py index 077eeff..5e6a45c 100644 --- a/Tests/unit/corelibs_applications/test_testapp.py +++ b/Tests/unit/corelibs_applications/test_testapp.py @@ -36,58 +36,6 @@ import Longbow.corelibs.exceptions as ex -@mock.patch('Longbow.corelibs.shellwrappers.sendtossh') -def test_testapp_nochecks1(m_sendtossh): - - """ - Test that if nochecks is set that no executable checks happen (check - boolean). - """ - - jobs = { - "jobone": { - "resource": "res1", - "executable": "exec1", - "nochecks": True - }, - "jobtwo": { - "resource": "res2", - "executable": "exec2", - "nochecks": True - } - } - - apps.testapp(jobs) - - assert m_sendtossh.call_count == 0 - - -@mock.patch('Longbow.corelibs.shellwrappers.sendtossh') -def test_testapp_nochecks2(m_sendtossh): - - """ - Test that if nochecks is set that no executable checks happen (check - string). - """ - - jobs = { - "jobone": { - "resource": "res1", - "executable": "exec1", - "nochecks": "true" - }, - "jobtwo": { - "resource": "res2", - "executable": "exec2", - "nochecks": "true" - } - } - - apps.testapp(jobs) - - assert m_sendtossh.call_count == 0 - - @mock.patch('Longbow.corelibs.shellwrappers.sendtossh') def test_testapp_exectest(m_sendtossh): From 9db62609ae0e996360d9d002408a4a5b993c71d3 Mon Sep 17 00:00:00 2001 From: jimboid Date: Fri, 9 Dec 2016 15:41:18 +0000 Subject: [PATCH 07/22] Removed redundant code --- Longbow/schedulers/lsf.py | 20 +++++++------------- Longbow/schedulers/pbs.py | 16 +++++----------- Longbow/schedulers/sge.py | 20 +++++++------------- Longbow/schedulers/slurm.py | 20 +++++++------------- Longbow/schedulers/soge.py | 20 +++++++------------- 5 files changed, 33 insertions(+), 63 deletions(-) diff --git a/Longbow/schedulers/lsf.py b/Longbow/schedulers/lsf.py index 282a0c7..37fad55 100755 --- a/Longbow/schedulers/lsf.py +++ b/Longbow/schedulers/lsf.py @@ -190,22 +190,16 @@ def status(job): stdout = shellout[0].split("\n") # Look up the job state and convert it to Longbow terminology. - try: - - # Now match the jobid against the list of jobs, extract the line and - # split it into a list - for line in stdout: - - line = line.split() + # Now match the jobid against the list of jobs, extract the line and + # split it into a list + for line in stdout: - if len(line) > 0 and job["jobid"] in line[0]: + line = line.split() - jobstate = states[line[2]] - break + if len(line) > 0 and job["jobid"] in line[0]: - except (IndexError, KeyError): - - jobstate = "Finished" + jobstate = states[line[2]] + break if jobstate == "": diff --git a/Longbow/schedulers/pbs.py b/Longbow/schedulers/pbs.py index 0f91334..61806dd 100755 --- a/Longbow/schedulers/pbs.py +++ b/Longbow/schedulers/pbs.py @@ -237,20 +237,14 @@ def status(job): # PBS will return a table, so split lines into a list. stdout = shellout[0].split("\n") - try: - - for line in stdout: - - line = line.split() + for line in stdout: - if len(line) > 0 and job["jobid"] in line[0]: + line = line.split() - jobstate = states[line[9]] - break + if len(line) > 0 and job["jobid"] in line[0]: - except (IndexError, KeyError): - - jobstate = "Finished" + jobstate = states[line[9]] + break if jobstate == "": diff --git a/Longbow/schedulers/sge.py b/Longbow/schedulers/sge.py index 53e86b5..e3884a4 100755 --- a/Longbow/schedulers/sge.py +++ b/Longbow/schedulers/sge.py @@ -181,22 +181,16 @@ def status(job): stdout = shellout[0].split("\n") # Look up the job state and convert it to Longbow terminology. - try: - - # Now match the jobid against the list of jobs, extract the line and - # split it into a list - for line in stdout: - - line = line.split() + # Now match the jobid against the list of jobs, extract the line and + # split it into a list + for line in stdout: - if len(line) > 0 and job["jobid"] in line[0]: + line = line.split() - jobstate = states[line[4]] - break + if len(line) > 0 and job["jobid"] in line[0]: - except (IndexError, KeyError): - - jobstate = "Finished" + jobstate = states[line[4]] + break if jobstate == "": diff --git a/Longbow/schedulers/slurm.py b/Longbow/schedulers/slurm.py index 6ea4b45..2cd81ca 100755 --- a/Longbow/schedulers/slurm.py +++ b/Longbow/schedulers/slurm.py @@ -196,22 +196,16 @@ def status(job): stdout = shellout[0].split("\n") # Look up the job state and convert it to Longbow terminology. - try: - - # Now match the jobid against the list of jobs, extract the line and - # split it into a list - for line in stdout: - - line = line.split() + # Now match the jobid against the list of jobs, extract the line and + # split it into a list + for line in stdout: - if len(line) > 0 and job["jobid"] in line[0]: + line = line.split() - jobstate = states[line[4]] - break + if len(line) > 0 and job["jobid"] in line[0]: - except (IndexError, KeyError): - - jobstate = "Finished" + jobstate = states[line[4]] + break if jobstate == "": diff --git a/Longbow/schedulers/soge.py b/Longbow/schedulers/soge.py index e9af283..ac48e4a 100755 --- a/Longbow/schedulers/soge.py +++ b/Longbow/schedulers/soge.py @@ -198,22 +198,16 @@ def status(job): stdout = shellout[0].split("\n") # Look up the job state and convert it to Longbow terminology. - try: - - # Now match the jobid against the list of jobs, extract the line and - # split it into a list - for line in stdout: - - line = line.split() + # Now match the jobid against the list of jobs, extract the line and + # split it into a list + for line in stdout: - if len(line) > 0 and job["jobid"] in line[0]: + line = line.split() - jobstate = states[line[4]] - break + if len(line) > 0 and job["jobid"] in line[0]: - except (IndexError, KeyError): - - jobstate = "Finished" + jobstate = states[line[4]] + break if jobstate == "": From bfd1d72a230a3718c4d900ec9dc3eecd87b6bb24 Mon Sep 17 00:00:00 2001 From: jimboid Date: Fri, 9 Dec 2016 15:55:43 +0000 Subject: [PATCH 08/22] renamed landscape.yml to include the dot --- landscape.yml => .landscape.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename landscape.yml => .landscape.yml (100%) diff --git a/landscape.yml b/.landscape.yml similarity index 100% rename from landscape.yml rename to .landscape.yml From 42b2b17c7bd48a6ebebeb295750cd1425954d6c0 Mon Sep 17 00:00:00 2001 From: jimboid Date: Fri, 9 Dec 2016 15:56:52 +0000 Subject: [PATCH 09/22] removed parens --- Longbow/corelibs/applications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Longbow/corelibs/applications.py b/Longbow/corelibs/applications.py index 6cc5f36..1bf66b4 100755 --- a/Longbow/corelibs/applications.py +++ b/Longbow/corelibs/applications.py @@ -74,7 +74,7 @@ def testapp(jobs): checked[jobs[job]["resource"]] = [] # Now check if we have tested this exec already. - if (jobs[job]["executable"] not in checked[jobs[job]["resource"]]): + if jobs[job]["executable"] not in checked[jobs[job]["resource"]]: # If not then add it to the list now. checked[jobs[job]["resource"]].extend([jobs[job]["executable"]]) From 94fb75ce23e7fcddeaa1a80951bd4082c2def18a Mon Sep 17 00:00:00 2001 From: jimboid Date: Thu, 5 Jan 2017 11:23:07 +0000 Subject: [PATCH 10/22] Attempt to fix issue #21 The freezing glitch occured if all jobs encountered submission errors, Longbow would just hang indefinitely. --- Longbow/corelibs/scheduling.py | 47 +++++++++++---- .../corelibs_scheduling/test_checkcomplete.py | 60 +++++++++++++++++-- 2 files changed, 89 insertions(+), 18 deletions(-) diff --git a/Longbow/corelibs/scheduling.py b/Longbow/corelibs/scheduling.py index 563f57f..fb17d3a 100755 --- a/Longbow/corelibs/scheduling.py +++ b/Longbow/corelibs/scheduling.py @@ -256,9 +256,23 @@ def monitor(jobs): LOG.warning("Could not write recovery file, possibly due to " "permissions on the ~/.Longbow directory.") - allfinished, allcomplete = _checkcomplete(jobs) + allcomplete, allfinished = _checkcomplete(jobs) - LOG.info("All jobs are complete.") + complete = 0 + error = 0 + + for job in jobs: + + if jobs[job]["laststatus"] == "Submit Error": + + error = error + 1 + + else: + + complete = complete + 1 + + LOG.info("Session complete - %s jobs ran - %s jobs encountered submission " + "errors.", complete, error) def prepare(jobs): @@ -584,17 +598,16 @@ def _stagejobfiles(jobs, save): for job in jobs: if (jobs[job]["laststatus"] == "Running" or - jobs[job]["laststatus"] == "Subjob(s) running"): + jobs[job]["laststatus"] == "Subjob(s) running" or + jobs[job]["laststatus"] == "Finished"): staging.stage_downstream(jobs[job]) - if jobs[job]["laststatus"] == "Finished": + if jobs[job]["laststatus"] == "Finished": - staging.stage_downstream(jobs[job]) + jobs[job]["laststatus"] = "Complete" - jobs[job]["laststatus"] = "Complete" - - save = True + save = True return save @@ -655,9 +668,10 @@ def _checkwaitingjobs(jobs, save): def _checkcomplete(jobs): """Check if all the jobs are complete.""" # Initialise variables - allfinished = False allcomplete = False + allfinished = False complete = [] + error = [] finished = [] for job in jobs: @@ -666,16 +680,25 @@ def _checkcomplete(jobs): complete.append(jobs[job]["laststatus"]) - if jobs[job]["laststatus"] != "Complete": + if (jobs[job]["laststatus"] != "Submit Error" and + jobs[job]["laststatus"] != "Complete"): + + finished.append(jobs[job]["laststatus"]) - finished.append(jobs[job]["laststatus"]) + if jobs[job]["laststatus"] == "Submit Error": + + error.append(jobs[job]["laststatus"]) if all(state == "Complete" for state in complete) and len(complete) != 0: allcomplete = True + if len(error) == len(jobs): + + allcomplete = True + if all(state == "Finished" for state in finished) and len(finished) != 0: allfinished = True - return allfinished, allcomplete + return allcomplete, allfinished diff --git a/Tests/unit/corelibs_scheduling/test_checkcomplete.py b/Tests/unit/corelibs_scheduling/test_checkcomplete.py index e656cdb..565f57e 100644 --- a/Tests/unit/corelibs_scheduling/test_checkcomplete.py +++ b/Tests/unit/corelibs_scheduling/test_checkcomplete.py @@ -38,7 +38,7 @@ def test_checkcomplete_single1(): } } - finished, complete = scheduling._checkcomplete(jobs) + complete, finished = scheduling._checkcomplete(jobs) assert finished is False assert complete is False @@ -56,7 +56,7 @@ def test_checkcomplete_single2(): } } - finished, complete = scheduling._checkcomplete(jobs) + complete, finished = scheduling._checkcomplete(jobs) assert finished is True assert complete is False @@ -74,7 +74,7 @@ def test_checkcomplete_single3(): } } - finished, complete = scheduling._checkcomplete(jobs) + complete, finished = scheduling._checkcomplete(jobs) assert finished is False assert complete is True @@ -98,7 +98,7 @@ def test_checkcomplete_multi1(): } } - finished, complete = scheduling._checkcomplete(jobs) + complete, finished = scheduling._checkcomplete(jobs) assert finished is False assert complete is False @@ -122,7 +122,7 @@ def test_checkcomplete_multi2(): } } - finished, complete = scheduling._checkcomplete(jobs) + complete, finished = scheduling._checkcomplete(jobs) assert finished is True assert complete is False @@ -146,7 +146,55 @@ def test_checkcomplete_multi3(): } } - finished, complete = scheduling._checkcomplete(jobs) + complete, finished = scheduling._checkcomplete(jobs) + + assert finished is False + assert complete is True + + +def test_checkcomplete_multi4(): + + """ + Check that submit error is ignored. + """ + + jobs = { + "jobone": { + "laststatus": "Submit Error" + }, + "jobtwo": { + "laststatus": "Complete" + }, + "jobthree": { + "laststatus": "Complete" + } + } + + complete, finished = scheduling._checkcomplete(jobs) + + assert finished is False + assert complete is True + + +def test_checkcomplete_multi5(): + + """ + Check that submit error is ignored. + """ + + jobs = { + "jobone": { + "laststatus": "Submit Error" + }, + "jobtwo": { + "laststatus": "Submit Error" + }, + "jobthree": { + "laststatus": "Submit Error" + } + } + + complete, finished = scheduling._checkcomplete(jobs) assert finished is False assert complete is True From 62e22be1b3e87737c66b7ed10e7eaf17eec6cf5b Mon Sep 17 00:00:00 2001 From: jimboid Date: Mon, 9 Jan 2017 15:13:37 +0000 Subject: [PATCH 11/22] update flags in example files --- Examples/how-to-run | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Examples/how-to-run b/Examples/how-to-run index c96b176..1b4e3f4 100755 --- a/Examples/how-to-run +++ b/Examples/how-to-run @@ -6,19 +6,19 @@ and learn how to submit a simple longbow job from their desktop using the below commands: LongbowExamples/QuickStart/Amber -longbow -verbose pmemd.MPI -O -i example.in -c example.min -p example.top -o example.out +longbow --verbose pmemd.MPI -O -i example.in -c example.min -p example.top -o example.out LongbowExamples/QuickStart/CHARMM -longbow -verbose charmm -i example.inp ">" example.out +longbow --verbose charmm -i example.inp ">" example.out LongbowExamples/QuickStart/Gromacs -longbow -verbose mdrun -s example.tpr -deffnm output +longbow --verbose mdrun -s example.tpr -deffnm output LongbowExamples/QuickStart/LAMMPS -longbow -verbose lmp_xc30 -i example.in -l output +longbow --verbose lmp_xc30 -i example.in -l output LongbowExamples/QuickStart/NAMD -longbow -verbose namd2 example.in ">" example.out +longbow --verbose namd2 example.in ">" example.out ################################################################################ LongbowExamples/ReplicateJob @@ -26,7 +26,7 @@ LongbowExamples/ReplicateJob In this directory, users can learn how to submit replicas by executing the below command on a desktop: -longbow -verbose -replicates 5 namd2 example.in +longbow --verbose -replicates 5 namd2 example.in ################################################################################ LongbowExamples/MultipleJobs @@ -37,7 +37,7 @@ In this directory, users can learn how to submit multiple jobs to multiple resources (HPC's). Edit /multi/job.conf to specify resources with entries in ~/.Longbow/hosts.conf and submit on your desktop using the below command: -longbow -job job.conf -verbose +longbow --job job.conf -verbose LongbowExamples/MultipleJobs/MultipleJobsDifferentApplications @@ -45,7 +45,7 @@ In this directory, users can learn how to submit multiple jobs that each use a different molecular dynamics package. Just execute the following command on a desktop: -longbow -job job.conf -verbose +longbow --job job.conf -verbose ################################################################################ LongbowExamples/CondorSubmission From 5d823fd038b3a29a018f15564251d99f667f83c8 Mon Sep 17 00:00:00 2001 From: jimboid Date: Mon, 9 Jan 2017 15:14:16 +0000 Subject: [PATCH 12/22] missed a flag --- Examples/how-to-run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/how-to-run b/Examples/how-to-run index 1b4e3f4..d6dd9b6 100755 --- a/Examples/how-to-run +++ b/Examples/how-to-run @@ -26,7 +26,7 @@ LongbowExamples/ReplicateJob In this directory, users can learn how to submit replicas by executing the below command on a desktop: -longbow --verbose -replicates 5 namd2 example.in +longbow --verbose --replicates 5 namd2 example.in ################################################################################ LongbowExamples/MultipleJobs From f65bfa67efd6539353fd445b683a6728cd669d79 Mon Sep 17 00:00:00 2001 From: jimboid Date: Tue, 10 Jan 2017 11:55:57 +0000 Subject: [PATCH 13/22] mistake in travis file --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 398010d..6843d45 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ python: - 3.5 # command to install dependencies before_install: - # Coveralls 4.0 doesn't support Python 3.2 + # Coverage 4.0+ doesn't support Python 3.2 - if [ "$TRAVIS_PYTHON_VERSION" == "3.2" ]; then travis_retry pip install coverage==3.7.1; fi - if [ "$TRAVIS_PYTHON_VERSION" != "3.2" ]; then travis_retry pip install coverage; fi - pip install coveralls From 4b7dcdf05f244115d66b13e15fc73a83092e085b Mon Sep 17 00:00:00 2001 From: jimboid Date: Tue, 10 Jan 2017 15:10:19 +0000 Subject: [PATCH 14/22] Corrected data access error on substitutions --- Longbow/corelibs/applications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Longbow/corelibs/applications.py b/Longbow/corelibs/applications.py index 1bf66b4..2d45d6c 100755 --- a/Longbow/corelibs/applications.py +++ b/Longbow/corelibs/applications.py @@ -170,7 +170,7 @@ def processjobs(jobs): substitution = getattr( apps, app.lower()).detectsubstitutions( - list(job["executableargs"])) + list(jobs[job]["executableargs"])) except AttributeError: From c157786be127dd0ad6d78108b92840d0a65fb924 Mon Sep 17 00:00:00 2001 From: jimboid Date: Tue, 10 Jan 2017 17:00:54 +0000 Subject: [PATCH 15/22] Command-line unrecognised Command-lines of the form: exec input.file > output.file would throw an error message to do with not being able to read them, this fix should resolve this. Maybe in future see if a generic solution can be found and tested. --- Longbow/corelibs/applications.py | 29 ++++++++++--------- .../test_proccommandline.py | 18 ++++++++++++ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/Longbow/corelibs/applications.py b/Longbow/corelibs/applications.py index 2d45d6c..d984e1b 100755 --- a/Longbow/corelibs/applications.py +++ b/Longbow/corelibs/applications.py @@ -250,31 +250,34 @@ def _proccommandline(job, filelist, foundflags, substitution): # Determine the command-line type and call the processor method. Start # with command-lines of the type exec < input.file. - if args[0] == "<" and len(args) > 1: + # Command-line type: exec input.file + if len(args) == 1 and args[0] != "<": - # Command-line type exec < input.file - foundflags = _procfiles(job, args[1], filelist, foundflags, + foundflags = _procfiles(job, args[0], filelist, foundflags, substitution) - elif len(args) == 1 and args[0] != "<": + # Command-line type: exec input.file > output.file + elif len(args) == 3 and args[1] == ">": - # Command-line type exec input.file foundflags = _procfiles(job, args[0], filelist, foundflags, substitution) - elif "-" in args[0]: + # Command-line type: exec < input.file + elif len(args) > 1 and args[0] == "<": - for arg in args: + foundflags = _procfiles(job, args[1], filelist, foundflags, + substitution) - # Command-line type exec -i file -c file - foundflags = _procfiles(job, arg, filelist, foundflags, - substitution) + # Command-line type exec -i file -c file + elif "-" in args[0] or "-" in args[1]: - elif "-" not in args[0] and "-" in args[1]: + # Sub executables. + if "-" not in args[0]: - for arg in args[1:]: + args = args[1:] + + for arg in args: - # Command-line type exec subexec -i file -c file foundflags = _procfiles(job, arg, filelist, foundflags, substitution) diff --git a/Tests/unit/corelibs_applications/test_proccommandline.py b/Tests/unit/corelibs_applications/test_proccommandline.py index 1ab0666..45b3e53 100644 --- a/Tests/unit/corelibs_applications/test_proccommandline.py +++ b/Tests/unit/corelibs_applications/test_proccommandline.py @@ -108,6 +108,24 @@ def test_proccommandline_test4(m_procfiles): assert m_procfiles.call_args[0][1] == "input.file" +@mock.patch('Longbow.corelibs.applications._procfiles') +def test_proccommandline_test5(m_procfiles): + + """ + Test that the correct method is selected based on the command-line. + """ + + job = { + "executable": "namd", + "executableargs": ["input.file", ">", "output.file"] + } + + apps._proccommandline(job, [], [], {}) + + assert m_procfiles.call_count == 1 + assert m_procfiles.call_args[0][1] == "input.file" + + def test_proccommandline_except(): """ From 4ea39a2453e07f363dea80799ad313dc85619520 Mon Sep 17 00:00:00 2001 From: jimboid Date: Wed, 11 Jan 2017 10:49:51 +0000 Subject: [PATCH 16/22] Fixes for command-line errors --- Longbow/corelibs/applications.py | 50 +++++-------------- .../test_proccommandline.py | 10 ++-- .../corelibs_applications/test_procfiles.py | 26 +++++++++- 3 files changed, 42 insertions(+), 44 deletions(-) diff --git a/Longbow/corelibs/applications.py b/Longbow/corelibs/applications.py index d984e1b..aa30178 100755 --- a/Longbow/corelibs/applications.py +++ b/Longbow/corelibs/applications.py @@ -244,47 +244,22 @@ def _proccommandline(job, filelist, foundflags, substitution): """ # Initialisation. + appplugins = getattr(apps, "PLUGINEXECS") + app = appplugins[job["executable"]] args = list(job["executableargs"]) + subexecs = getattr( + apps, app.lower()).EXECDATA[job["executable"]]["subexecutables"] try: - # Determine the command-line type and call the processor method. Start - # with command-lines of the type exec < input.file. - # Command-line type: exec input.file - if len(args) == 1 and args[0] != "<": - - foundflags = _procfiles(job, args[0], filelist, foundflags, - substitution) - - # Command-line type: exec input.file > output.file - elif len(args) == 3 and args[1] == ">": - - foundflags = _procfiles(job, args[0], filelist, foundflags, - substitution) - - # Command-line type: exec < input.file - elif len(args) > 1 and args[0] == "<": - - foundflags = _procfiles(job, args[1], filelist, foundflags, - substitution) - - # Command-line type exec -i file -c file - elif "-" in args[0] or "-" in args[1]: - - # Sub executables. - if "-" not in args[0]: - - args = args[1:] + for arg in args: - for arg in args: + if (arg != "<" and arg != ">" and arg[0] != "-" and + arg not in subexecs): foundflags = _procfiles(job, arg, filelist, foundflags, substitution) - else: - - raise ValueError - except (IndexError, ValueError): raise exceptions.RequiredinputError( @@ -329,16 +304,15 @@ def _procfiles(job, arg, filelist, foundflags, substitution): # If we have a valid file if os.path.isfile(os.path.join(job["localworkdir"], fileitem)): - # Mark files as found. - if (len(initargs) > 1 and initargs[initargs.index(arg) - 1] not in - foundflags): + if arg == initargs[0]: - foundflags.append(initargs[initargs.index(arg) - 1]) + foundflags.append("<") - elif(len(initargs) == 1 and initargs[initargs.index(arg) - 1] not + # Mark files as found. + elif (len(initargs) > 1 and initargs[initargs.index(arg) - 1] not in foundflags): - foundflags.append("<") + foundflags.append(initargs[initargs.index(arg) - 1]) # Search input file for any file dependencies. try: diff --git a/Tests/unit/corelibs_applications/test_proccommandline.py b/Tests/unit/corelibs_applications/test_proccommandline.py index 45b3e53..9c4d508 100644 --- a/Tests/unit/corelibs_applications/test_proccommandline.py +++ b/Tests/unit/corelibs_applications/test_proccommandline.py @@ -68,7 +68,7 @@ def test_proccommandline_test2(m_procfiles): apps._proccommandline(job, [], [], {}) - assert m_procfiles.call_count == 6 + assert m_procfiles.call_count == 3 assert m_procfiles.call_args[0][1] == "file" @@ -86,7 +86,7 @@ def test_proccommandline_test3(m_procfiles): apps._proccommandline(job, [], [], {}) - assert m_procfiles.call_count == 2 + assert m_procfiles.call_count == 1 assert m_procfiles.call_args[0][1] == "filename" @@ -116,14 +116,14 @@ def test_proccommandline_test5(m_procfiles): """ job = { - "executable": "namd", + "executable": "namd2", "executableargs": ["input.file", ">", "output.file"] } apps._proccommandline(job, [], [], {}) - assert m_procfiles.call_count == 1 - assert m_procfiles.call_args[0][1] == "input.file" + assert m_procfiles.call_count == 2 + assert m_procfiles.call_args[0][1] == "output.file" def test_proccommandline_except(): diff --git a/Tests/unit/corelibs_applications/test_procfiles.py b/Tests/unit/corelibs_applications/test_procfiles.py index a7e127a..1164531 100644 --- a/Tests/unit/corelibs_applications/test_procfiles.py +++ b/Tests/unit/corelibs_applications/test_procfiles.py @@ -97,7 +97,7 @@ def test_procfiles_gromacs(): assert filelist == ["test.tpr"] -def test_procfiles_namd(): +def test_procfiles_namd1(): """ Test to make sure that the file and flag is picked up for an namd-like @@ -121,6 +121,30 @@ def test_procfiles_namd(): assert filelist == ["input"] +def test_procfiles_namd2(): + + """ + Test to make sure that the file and flag is picked up for an namd-like + command-line. + """ + + arg = "input" + filelist = [] + foundflags = [] + job = { + "executable": "namd2", + "replicates": "1", + "localworkdir": "Tests/standards/jobs/single", + "executableargs": ["input", ">", "output"] + } + substitution = {} + + foundflags = apps._procfiles(job, arg, filelist, foundflags, substitution) + + assert foundflags == ["<"] + assert filelist == ["input"] + + def test_procfiles_reps(): """ From 4ed8b017717a90c4c64542760314580ee6465c89 Mon Sep 17 00:00:00 2001 From: jimboid Date: Wed, 11 Jan 2017 11:15:16 +0000 Subject: [PATCH 17/22] updated examples --- Examples/how-to-run | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Examples/how-to-run b/Examples/how-to-run index d6dd9b6..d744b45 100755 --- a/Examples/how-to-run +++ b/Examples/how-to-run @@ -37,7 +37,7 @@ In this directory, users can learn how to submit multiple jobs to multiple resources (HPC's). Edit /multi/job.conf to specify resources with entries in ~/.Longbow/hosts.conf and submit on your desktop using the below command: -longbow --job job.conf -verbose +longbow --job job.conf --verbose LongbowExamples/MultipleJobs/MultipleJobsDifferentApplications @@ -45,7 +45,7 @@ In this directory, users can learn how to submit multiple jobs that each use a different molecular dynamics package. Just execute the following command on a desktop: -longbow --job job.conf -verbose +longbow --job job.conf --verbose ################################################################################ LongbowExamples/CondorSubmission From 85dd6f1b9a937a9c7ce09656c1dc7ec64f8c2848 Mon Sep 17 00:00:00 2001 From: jimboid Date: Wed, 11 Jan 2017 11:44:36 +0000 Subject: [PATCH 18/22] Changed parameter name Changed parameter frequency to polling-frequency to match syntax of staging-frequency --- Longbow/corelibs/configuration.py | 10 +++++----- Longbow/corelibs/scheduling.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Longbow/corelibs/configuration.py b/Longbow/corelibs/configuration.py index b26f0d8..26e40d3 100755 --- a/Longbow/corelibs/configuration.py +++ b/Longbow/corelibs/configuration.py @@ -76,23 +76,23 @@ "email-flags": "", "executable": "", "executableargs": "", - "frequency": "300", "handler": "", "host": "", "localworkdir": "", "modules": "", "maxtime": "24:00", "memory": "", - "scripts": "", - "staging-frequency": "300", - "sge-peflag": "mpi", - "sge-peoverride": "false", + "polling-frequency": "300", "port": "22", "queue": "", "remoteworkdir": "", "resource": "", "replicates": "1", "scheduler": "", + "scripts": "", + "staging-frequency": "300", + "sge-peflag": "mpi", + "sge-peoverride": "false", "user": "", "upload-exclude": "", "upload-include": "" diff --git a/Longbow/corelibs/scheduling.py b/Longbow/corelibs/scheduling.py index fb17d3a..9b81003 100755 --- a/Longbow/corelibs/scheduling.py +++ b/Longbow/corelibs/scheduling.py @@ -527,9 +527,9 @@ def _monitorinitialise(jobs): stageinterval = int(jobs[job]["staging-frequency"]) # Attempt to grab a polling frequency that might have been set - if pollinterval < int(jobs[job]["frequency"]): + if pollinterval < int(jobs[job]["polling-frequency"]): - pollinterval = int(jobs[job]["frequency"]) + pollinterval = int(jobs[job]["polling-frequency"]) # If somehow the polling interval parameter is still zero, reduce the # polling to once every 5 minutes. From b53f2c0a401ea53c9c972b2fa9ba8e75e3060a6e Mon Sep 17 00:00:00 2001 From: jimboid Date: Wed, 11 Jan 2017 15:55:29 +0000 Subject: [PATCH 19/22] More fixes Multijobs still busted though --- Longbow/corelibs/configuration.py | 19 +------ Longbow/corelibs/entrypoints.py | 4 +- .../test_processconfigs.py | 14 ++--- .../test_processconfigsparams.py | 16 ++++-- .../test_processconfigsresource.py | 18 +++---- .../test_commandlineproc.py | 52 +++++++++---------- Tests/unit/corelibs_entrypoints/test_main.py | 16 +++--- .../test_monitorinitialise.py | 8 +-- 8 files changed, 67 insertions(+), 80 deletions(-) diff --git a/Longbow/corelibs/configuration.py b/Longbow/corelibs/configuration.py index 26e40d3..22b1029 100755 --- a/Longbow/corelibs/configuration.py +++ b/Longbow/corelibs/configuration.py @@ -427,22 +427,7 @@ def _processconfigsfinalinit(jobs): jobs[job]["localworkdir"] = os.getcwd() - # Fix for python 3 where basestring is now str. - try: - - # If the exec arguments are in string form, split to list. - if isinstance(jobs[job]["executableargs"], basestring): - - jobs[job]["executableargs"] = ( - jobs[job]["executableargs"].split()) - - except NameError: - - # If the exec arguments are in string form, split to list. - if isinstance(jobs[job]["executableargs"], str): - - jobs[job]["executableargs"] = ( - jobs[job]["executableargs"].split()) + jobs[job]["executableargs"] = jobs[job]["executableargs"].split() # If modules hasn't been set then try and use a default. if jobs[job]["modules"] is "": @@ -467,7 +452,7 @@ def _processconfigsparams(jobs, parameters, jobdata, hostdata): for item in jobs[job]: # This should already be dealt with. - if item is not "resource": + if item != "resource": # Command-line overrides are highest priority. if item in parameters and parameters[item] is not "": diff --git a/Longbow/corelibs/entrypoints.py b/Longbow/corelibs/entrypoints.py index 9dedcc7..123ce23 100644 --- a/Longbow/corelibs/entrypoints.py +++ b/Longbow/corelibs/entrypoints.py @@ -90,7 +90,7 @@ def main(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -450,7 +450,7 @@ def _commandlineproc(alllongbowargs, cmdlnargs, parameters): "Recognised arguments are: {1}".format(item, allowedargs)) parameters["executable"] = executable - parameters["executableargs"] = execargs + parameters["executableargs"] = " ".join(execargs) return longbowargs diff --git a/Tests/unit/corelibs_configuration/test_processconfigs.py b/Tests/unit/corelibs_configuration/test_processconfigs.py index 6d2323f..f373b0e 100644 --- a/Tests/unit/corelibs_configuration/test_processconfigs.py +++ b/Tests/unit/corelibs_configuration/test_processconfigs.py @@ -39,7 +39,7 @@ def test_processconfigs_test1(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "/tmp/hostfile.conf", "job": "", "jobname": "", @@ -67,7 +67,7 @@ def test_processconfigs_test2(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": conffile, "job": "/tmp/jobfile.conf", "jobname": "", @@ -95,8 +95,8 @@ def test_processconfigs_test3(): "debug": False, "disconnect": False, "executable": "pmemd.MPI", - "executableargs": ["-O", "-i", "example.in", "-c", "example.min", "-p", - "example.top", "-o", "example.out"], + "executableargs": + "-O -i example.in -c example.min -p example.top -o example.out", "hosts": conffile, "job": "", "jobname": "", @@ -140,8 +140,8 @@ def test_processconfigs_test4(): "debug": False, "disconnect": False, "executable": "pmemd.MPI", - "executableargs": ["-O", "-i", "example.in", "-c", "example.min", "-p", - "example.top", "-o", "example.out"], + "executableargs": + "-O -i example.in -c example.min -p example.top -o example.out", "hosts": conffile, "job": "", "jobname": "test-job", @@ -186,7 +186,7 @@ def test_processconfigs_test5(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": hostfile, "job": jobfile, "jobname": "", diff --git a/Tests/unit/corelibs_configuration/test_processconfigsparams.py b/Tests/unit/corelibs_configuration/test_processconfigsparams.py index ab08a8f..6fc2f06 100644 --- a/Tests/unit/corelibs_configuration/test_processconfigsparams.py +++ b/Tests/unit/corelibs_configuration/test_processconfigsparams.py @@ -104,7 +104,7 @@ def test_processconfigsparams_test1(): "debug": False, "disconnect": False, "executable": "test.exec", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -221,7 +221,7 @@ def test_processconfigsparams_test2(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -246,17 +246,23 @@ def test_processconfigsparams_test2(): jobdata = { "jobone": { - "executable": "job1.exec" + "executable": "job1.exec", + "executableargs": "-i example.in -c example.rst -p example.top" }, "jobtwo": { - "executable": "job2.exec" + "executable": "job2.exec", + "executableargs": "-i example.in -c example.rst -p example.top" } } conf._processconfigsparams(jobs, parameters, jobdata, hostdata) assert jobs["jobone"]["executable"] == "job1.exec" + assert jobs["jobone"]["executableargs"] == \ + "-i example.in -c example.rst -p example.top" assert jobs["jobtwo"]["executable"] == "job2.exec" + assert jobs["jobtwo"]["executableargs"] == \ + "-i example.in -c example.rst -p example.top" def test_processconfigsparams_test3(): @@ -338,7 +344,7 @@ def test_processconfigsparams_test3(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", diff --git a/Tests/unit/corelibs_configuration/test_processconfigsresource.py b/Tests/unit/corelibs_configuration/test_processconfigsresource.py index 0061029..d070484 100644 --- a/Tests/unit/corelibs_configuration/test_processconfigsresource.py +++ b/Tests/unit/corelibs_configuration/test_processconfigsresource.py @@ -62,7 +62,7 @@ def test_processconfigsresource1(): "email-flags": "", "executable": "", "executableargs": "", - "frequency": "", + "polling-frequency": "", "handler": "", "host": "", "localworkdir": "", @@ -98,7 +98,6 @@ def test_processconfigsresource1(): "email-flags": "", "executable": "", "executableargs": "", - "frequency": "300", "handler": "", "host": "", "localworkdir": "", @@ -110,6 +109,7 @@ def test_processconfigsresource1(): "staging-frequency": "300", "sge-peflag": "mpi", "sge-peoverride": "false", + "polling-frequency": "300", "port": "22", "queue": "", "remoteworkdir": "", @@ -165,7 +165,7 @@ def test_processconfigsresource2(): "email-flags": "", "executable": "", "executableargs": "", - "frequency": "", + "polling-frequency": "", "handler": "", "host": "", "localworkdir": "", @@ -201,7 +201,7 @@ def test_processconfigsresource2(): "email-flags": "", "executable": "", "executableargs": "", - "frequency": "300", + "polling-frequency": "300", "handler": "", "host": "", "localworkdir": "", @@ -270,7 +270,7 @@ def test_processconfigsresource3(): "email-flags": "", "executable": "", "executableargs": "", - "frequency": "", + "polling-frequency": "", "handler": "", "host": "", "localworkdir": "", @@ -305,7 +305,7 @@ def test_processconfigsresource3(): "email-flags": "", "executable": "", "executableargs": "", - "frequency": "300", + "polling-frequency": "300", "handler": "", "host": "", "localworkdir": "", @@ -372,7 +372,7 @@ def test_processconfigsresource4(): "email-flags": "", "executable": "", "executableargs": "", - "frequency": "", + "polling-frequency": "", "handler": "", "host": "", "localworkdir": "", @@ -408,7 +408,7 @@ def test_processconfigsresource4(): "email-flags": "", "executable": "", "executableargs": "", - "frequency": "300", + "polling-frequency": "300", "handler": "", "host": "", "localworkdir": "", @@ -475,7 +475,7 @@ def test_processconfigsresource5(): "email-flags": "", "executable": "", "executableargs": "", - "frequency": "", + "polling-frequency": "", "handler": "", "host": "", "localworkdir": "", diff --git a/Tests/unit/corelibs_entrypoints/test_commandlineproc.py b/Tests/unit/corelibs_entrypoints/test_commandlineproc.py index 3170637..d993c44 100644 --- a/Tests/unit/corelibs_entrypoints/test_commandlineproc.py +++ b/Tests/unit/corelibs_entrypoints/test_commandlineproc.py @@ -72,7 +72,7 @@ def test_cmdlineproc_test1(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -89,7 +89,7 @@ def test_cmdlineproc_test1(): parameters) assert parameters["executable"] == "" - assert parameters["executableargs"] == [] + assert parameters["executableargs"] == "" assert longbowargs == [] @@ -101,7 +101,7 @@ def test_cmdlineproc_test2(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -118,7 +118,7 @@ def test_cmdlineproc_test2(): parameters) assert parameters["executable"] == "" - assert parameters["executableargs"] == [] + assert parameters["executableargs"] == "" assert longbowargs == ["-about"] @@ -130,7 +130,7 @@ def test_cmdlineproc_test3(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -147,7 +147,7 @@ def test_cmdlineproc_test3(): parameters) assert parameters["executable"] == "" - assert parameters["executableargs"] == [] + assert parameters["executableargs"] == "" assert longbowargs == ["--about"] @@ -159,7 +159,7 @@ def test_cmdlineproc_test4(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -177,7 +177,7 @@ def test_cmdlineproc_test4(): parameters) assert parameters["executable"] == "" - assert parameters["executableargs"] == [] + assert parameters["executableargs"] == "" assert longbowargs == ["--hosts", "hosts.file", "--jobname", "test", "--replicates", "1000", "--disconnect"] @@ -190,7 +190,7 @@ def test_cmdlineproc_test5(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -208,7 +208,7 @@ def test_cmdlineproc_test5(): parameters) assert parameters["executable"] == "pmemd.MPI" - assert parameters["executableargs"] == [] + assert parameters["executableargs"] == "" assert longbowargs == ["--hosts", "hosts.file", "--jobname", "test", "--replicates", "1000", "--disconnect"] @@ -221,7 +221,7 @@ def test_cmdlineproc_test6(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -241,9 +241,8 @@ def test_cmdlineproc_test6(): parameters) assert parameters["executable"] == "pmemd.MPI" - assert parameters["executableargs"] == ["-O", "-i", "ex.in", "-c", - "ex.min", "-p", "ex.top", - "-o", "ex.out"] + assert parameters["executableargs"] == \ + "-O -i ex.in -c ex.min -p ex.top -o ex.out" assert longbowargs == ["--hosts", "hosts.file", "--jobname", "test", "--replicates", "1000", "--disconnect"] @@ -256,7 +255,7 @@ def test_cmdlineproc_test7(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -274,7 +273,7 @@ def test_cmdlineproc_test7(): parameters) assert parameters["executable"] == "test.exe" - assert parameters["executableargs"] == [] + assert parameters["executableargs"] == "" assert longbowargs == ["--hosts", "hosts.file", "--jobname", "test", "--replicates", "1000", "--disconnect"] @@ -289,7 +288,7 @@ def test_cmdlineproc_test8(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -308,8 +307,7 @@ def test_cmdlineproc_test8(): parameters) assert parameters["executable"] == "test.exe" - assert parameters["executableargs"] == ["-i", "input.file", "param1", - "--someflag"] + assert parameters["executableargs"] == "-i input.file param1 --someflag" assert longbowargs == ["--hosts", "hosts.file", "--jobname", "test", "--replicates", "1000", "--disconnect"] @@ -325,7 +323,7 @@ def test_cmdlineproc_test9(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -344,8 +342,7 @@ def test_cmdlineproc_test9(): parameters) assert parameters["executable"] == "test.exe" - assert parameters["executableargs"] == ["-i", "input.file", "param1", - "--someflag"] + assert parameters["executableargs"] == "-i input.file param1 --someflag" assert longbowargs == ["--hosts", "hosts.file", "--jobname", "test", "--disconnect", "--replicates", "1000"] @@ -358,7 +355,7 @@ def test_cmdlineproc_test10(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -375,7 +372,7 @@ def test_cmdlineproc_test10(): parameters) assert parameters["executable"] == "test.exe" - assert parameters["executableargs"] == [] + assert parameters["executableargs"] == "" assert longbowargs == [] @@ -387,7 +384,7 @@ def test_cmdlineproc_test11(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", @@ -404,8 +401,7 @@ def test_cmdlineproc_test11(): parameters) assert parameters["executable"] == "test.exe" - assert parameters["executableargs"] == ["-i", "input.file", "param1", - "--someflag"] + assert parameters["executableargs"] == "-i input.file param1 --someflag" assert longbowargs == [] @@ -417,7 +413,7 @@ def test_cmdlineproc_test12(): "debug": False, "disconnect": False, "executable": "", - "executableargs": [], + "executableargs": "", "hosts": "", "job": "", "jobname": "", diff --git a/Tests/unit/corelibs_entrypoints/test_main.py b/Tests/unit/corelibs_entrypoints/test_main.py index 30bbe1f..b47a4f0 100644 --- a/Tests/unit/corelibs_entrypoints/test_main.py +++ b/Tests/unit/corelibs_entrypoints/test_main.py @@ -61,8 +61,8 @@ def test_main_test1(m_isfile, m_longbowmain): assert params["debug"] is False assert params["disconnect"] is False assert params["executable"] == "pmemd.MPI" - assert params["executableargs"] == ["-O", "-i", "ex.in", "-c", "ex.min", - "-p", "ex.top", "-o", "ex.out"] + assert params["executableargs"] == \ + "-O -i ex.in -c ex.min -p ex.top -o ex.out" assert params["hosts"] == os.path.join(os.getcwd(), "hosts.conf") assert params["job"] == "" assert params["jobname"] == "testjob" @@ -98,8 +98,8 @@ def test_main_test2(m_isfile, m_longbowmain): assert params["debug"] is False assert params["disconnect"] is False assert params["executable"] == "pmemd.MPI" - assert params["executableargs"] == ["-O", "-i", "ex.in", "-c", "ex.min", - "-p", "ex.top", "-o", "ex.out"] + assert params["executableargs"] == \ + "-O -i ex.in -c ex.min -p ex.top -o ex.out" assert params["hosts"] == os.path.join(os.getcwd(), "hosts.conf") assert params["job"] == "" assert params["jobname"] == "testjob" @@ -159,8 +159,8 @@ def test_main_test4(m_isfile, m_longbowmain): assert params["debug"] is False assert params["disconnect"] is False assert params["executable"] == "pmemd.MPI" - assert params["executableargs"] == ["-O", "-i", "ex.in", "-c", "ex.min", - "-p", "ex.top", "-o", "ex.out"] + assert params["executableargs"] == \ + "-O -i ex.in -c ex.min -p ex.top -o ex.out" assert params["hosts"] == os.path.join(os.getcwd(), "hosts.conf") assert params["job"] == "" assert params["jobname"] == "testjob" @@ -197,8 +197,8 @@ def test_main_test5(m_isfile, m_longbowmain): assert params["debug"] is True assert params["disconnect"] is False assert params["executable"] == "pmemd.MPI" - assert params["executableargs"] == ["-O", "-i", "ex.in", "-c", "ex.min", - "-p", "ex.top", "-o", "ex.out"] + assert params["executableargs"] == \ + "-O -i ex.in -c ex.min -p ex.top -o ex.out" assert params["hosts"] == os.path.join(os.getcwd(), "hosts.conf") assert params["job"] == "" assert params["jobname"] == "testjob" diff --git a/Tests/unit/corelibs_scheduling/test_monitorinitialise.py b/Tests/unit/corelibs_scheduling/test_monitorinitialise.py index fd80e66..e9bb1db 100644 --- a/Tests/unit/corelibs_scheduling/test_monitorinitialise.py +++ b/Tests/unit/corelibs_scheduling/test_monitorinitialise.py @@ -38,14 +38,14 @@ def test_monitorinitialise_test1(): "queue-max": "0", "queue-slots": "0", "staging-frequency": "0", - "frequency": "0" + "polling-frequency": "0" }, "jobtwo": { "resource": "test-machine", "queue-max": "0", "queue-slots": "0", "staging-frequency": "0", - "frequency": "0" + "polling-frequency": "0" } } @@ -67,14 +67,14 @@ def test_monitorinitialise_test2(): "queue-max": "0", "queue-slots": "0", "staging-frequency": "100", - "frequency": "400" + "polling-frequency": "400" }, "jobtwo": { "resource": "test-machine3", "queue-max": "0", "queue-slots": "0", "staging-frequency": "0", - "frequency": "0" + "polling-frequency": "0" } } From 1549d36bac4899b04c6a54376376d0ffec34b66e Mon Sep 17 00:00:00 2001 From: jimboid Date: Wed, 11 Jan 2017 16:47:33 +0000 Subject: [PATCH 20/22] Fix for busted multijobs and globals files --- Longbow/corelibs/applications.py | 37 ++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/Longbow/corelibs/applications.py b/Longbow/corelibs/applications.py index aa30178..8377af9 100755 --- a/Longbow/corelibs/applications.py +++ b/Longbow/corelibs/applications.py @@ -237,6 +237,33 @@ def _flagvalidator(job, foundflags): .format(job["jobname"], flags, app)) +def _markfoundfiles(arg, initargs, foundflags): + """Method to mark file flags as found.""" + try: + + pos = initargs.index(arg) - 1 + + except ValueError: + + pos = initargs.index("../" + arg) - 1 + + # In cases where there is a single input file as the first parameter. This + # should cover cases such as: + # exec input.file + # exec input.file > output.file + if arg == initargs[0]: + + foundflags.append("<") + + # All other cases should pretty much be formats like: + # exec -flag file -flag file -flag file + elif len(initargs) > 1 and initargs[pos] not in foundflags: + + foundflags.append(initargs[pos]) + + return foundflags + + def _proccommandline(job, filelist, foundflags, substitution): """Command-line processor. @@ -304,15 +331,7 @@ def _procfiles(job, arg, filelist, foundflags, substitution): # If we have a valid file if os.path.isfile(os.path.join(job["localworkdir"], fileitem)): - if arg == initargs[0]: - - foundflags.append("<") - - # Mark files as found. - elif (len(initargs) > 1 and initargs[initargs.index(arg) - 1] not - in foundflags): - - foundflags.append(initargs[initargs.index(arg) - 1]) + _markfoundfiles(arg, initargs, foundflags) # Search input file for any file dependencies. try: From 242b7b0c8f66c5aa3943a3d27bf75793ad9fc005 Mon Sep 17 00:00:00 2001 From: jimboid Date: Thu, 12 Jan 2017 09:28:09 +0000 Subject: [PATCH 21/22] Updated example files to work with latest Longbow --- .../MultipleJobs/MultipleJobTypes/job.conf | 5 --- .../job.conf | 41 +++++++++---------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/Examples/MultipleJobs/MultipleJobTypes/job.conf b/Examples/MultipleJobs/MultipleJobTypes/job.conf index 100a22f..2d8daa6 100644 --- a/Examples/MultipleJobs/MultipleJobTypes/job.conf +++ b/Examples/MultipleJobs/MultipleJobTypes/job.conf @@ -12,8 +12,3 @@ executable = pmemd.MPI maxtime = 00:10 executableargs = -i example.in -c example.rst -p example.top -o example.out replicates = 5 - - - - - diff --git a/Examples/MultipleJobs/MultipleJobsDifferentApplications/job.conf b/Examples/MultipleJobs/MultipleJobsDifferentApplications/job.conf index c0ae279..16517a3 100644 --- a/Examples/MultipleJobs/MultipleJobsDifferentApplications/job.conf +++ b/Examples/MultipleJobs/MultipleJobsDifferentApplications/job.conf @@ -1,45 +1,44 @@ [amber] -executable = pmemd.MPI resource = Archer -maxtime = 00:10 +maxtime = 00:15 cores = 24 -frequency = 60 -modules = amber +polling-frequency = 60 +staging-frequency = 120 +executable = pmemd.MPI executableargs = -i example.in -c example.rst -p example.top -o example.out [gromacs_s] -executable = mdrun_mpi resource = Archer -maxtime = 00:10 +maxtime = 00:15 cores = 24 -frequency = 60 -modules = gromacs +polling-frequency = 60 +staging-frequency = 120 +executable = mdrun_mpi executableargs = -deffnm example [gromacs_d] -executable = mdrun_mpi_d resource = Archer -maxtime = 00:10 +maxtime = 00:15 cores = 24 -frequency = 60 -modules = gromacs +polling-frequency = 60 +staging-frequency = 120 +executable = mdrun_mpi_d executableargs = -deffnm example [namd] -executable = namd2 resource = Archer -maxtime = 00:10 +maxtime = 00:15 cores = 24 -frequency = 60 -modules = namd +polling-frequency = 60 +staging-frequency = 120 +executable = namd2 executableargs = example.in > example.out [lammps] -executable = lmp_xc30 resource = Archer -maxtime = 00:10 +maxtime = 00:15 cores = 24 -frequency = 60 -modules = lammps/lammps-9Dec14 +polling-frequency = 60 +staging-frequency = 120 +executable = lmp_xc30 executableargs = -i example.in -sf opt - From 978d0d4e2370b1fb98412c738e9df18bd0bf9112 Mon Sep 17 00:00:00 2001 From: jimboid Date: Thu, 12 Jan 2017 11:14:02 +0000 Subject: [PATCH 22/22] update tests for coverage --- .../corelibs_applications/test_procfiles.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/Tests/unit/corelibs_applications/test_procfiles.py b/Tests/unit/corelibs_applications/test_procfiles.py index 1164531..981fa4d 100644 --- a/Tests/unit/corelibs_applications/test_procfiles.py +++ b/Tests/unit/corelibs_applications/test_procfiles.py @@ -145,7 +145,7 @@ def test_procfiles_namd2(): assert filelist == ["input"] -def test_procfiles_reps(): +def test_procfiles_reps1(): """ Test for replicate variant. @@ -167,3 +167,26 @@ def test_procfiles_reps(): assert foundflags == ["-c"] assert filelist == ["rep1", "rep1/coords", "rep2", "rep2/coords", "rep3", "rep3/coords"] + + +def test_procfiles_reps2(): + + """ + Test for replicate variant with global. + """ + + arg = "topol" + filelist = [] + foundflags = [] + job = { + "executable": "pmemd.MPI", + "replicates": "3", + "localworkdir": "Tests/standards/jobs/replicate", + "executableargs": ["-i", "input", "-c", "coords", "-p", "topol"] + } + substitution = {} + + foundflags = apps._procfiles(job, arg, filelist, foundflags, substitution) + + assert foundflags == ["-p"] + assert filelist == ["rep1", "topol", "rep2", "rep3"]