Merge branch 'development'

.travis.yml

@@ -8,7 +8,7 @@ python:
   - 3.5
 # command to install dependencies
 before_install:
-  # Coveralls 4.0 doesn't support Python 3.2
+  # Coverage 4.0+ doesn't support Python 3.2
   - if [ "$TRAVIS_PYTHON_VERSION" == "3.2" ]; then travis_retry pip install coverage==3.7.1; fi
   - if [ "$TRAVIS_PYTHON_VERSION" != "3.2" ]; then travis_retry pip install coverage; fi
   - pip install coveralls

Examples/MultipleJobs/MultipleJobTypes/job.conf

@@ -12,8 +12,3 @@ executable = pmemd.MPI
 maxtime = 00:10
 executableargs = -i example.in -c example.rst -p example.top -o example.out
 replicates = 5
-
-
-
-
-

Examples/MultipleJobs/MultipleJobsDifferentApplications/job.conf

@@ -1,45 +1,44 @@
 [amber]
-executable = pmemd.MPI
 resource = Archer
-maxtime = 00:10
+maxtime = 00:15
 cores = 24
-frequency = 60
-modules = amber
+polling-frequency = 60
+staging-frequency = 120
+executable = pmemd.MPI
 executableargs = -i example.in -c example.rst -p example.top -o example.out
 
 [gromacs_s]
-executable = mdrun_mpi
 resource = Archer
-maxtime = 00:10
+maxtime = 00:15
 cores = 24
-frequency = 60
-modules = gromacs
+polling-frequency = 60
+staging-frequency = 120
+executable = mdrun_mpi
 executableargs = -deffnm example
 
 [gromacs_d]
-executable = mdrun_mpi_d
 resource = Archer
-maxtime = 00:10
+maxtime = 00:15
 cores = 24
-frequency = 60
-modules = gromacs
+polling-frequency = 60
+staging-frequency = 120
+executable = mdrun_mpi_d
 executableargs = -deffnm example
 
 [namd]
-executable = namd2
 resource = Archer
-maxtime = 00:10
+maxtime = 00:15
 cores = 24
-frequency = 60
-modules = namd
+polling-frequency = 60
+staging-frequency = 120
+executable = namd2
 executableargs = example.in > example.out
 
 [lammps]
-executable = lmp_xc30
 resource = Archer
-maxtime = 00:10
+maxtime = 00:15
 cores = 24
-frequency = 60
-modules = lammps/lammps-9Dec14
+polling-frequency = 60
+staging-frequency = 120
+executable = lmp_xc30
 executableargs = -i example.in -sf opt
-

Examples/how-to-run

@@ -6,27 +6,27 @@ and learn how to submit a simple longbow job from their desktop using the below
 commands:
 
 LongbowExamples/QuickStart/Amber
-longbow -verbose pmemd.MPI -O -i example.in -c example.min -p example.top -o example.out
+longbow --verbose pmemd.MPI -O -i example.in -c example.min -p example.top -o example.out
 
 LongbowExamples/QuickStart/CHARMM
-longbow -verbose charmm -i example.inp ">" example.out
+longbow --verbose charmm -i example.inp ">" example.out
 
 LongbowExamples/QuickStart/Gromacs
-longbow -verbose mdrun -s example.tpr -deffnm output 
+longbow --verbose mdrun -s example.tpr -deffnm output 
 
 LongbowExamples/QuickStart/LAMMPS
-longbow -verbose lmp_xc30 -i example.in -l output
+longbow --verbose lmp_xc30 -i example.in -l output
 
 LongbowExamples/QuickStart/NAMD
-longbow -verbose namd2 example.in ">" example.out
+longbow --verbose namd2 example.in ">" example.out
 
 ################################################################################
 LongbowExamples/ReplicateJob
 
 In this directory, users can learn how to submit replicas by executing the below 
 command on a desktop:
 
-longbow -verbose -replicates 5 namd2 example.in
+longbow --verbose --replicates 5 namd2 example.in
 
 ################################################################################
 LongbowExamples/MultipleJobs
@@ -37,15 +37,15 @@ In this directory, users can learn how to submit multiple jobs to multiple
 resources (HPC's). Edit /multi/job.conf to specify resources with entries in 
 ~/.Longbow/hosts.conf and submit on your desktop using the below command:
 
-longbow -job job.conf -verbose
+longbow --job job.conf --verbose
 
 LongbowExamples/MultipleJobs/MultipleJobsDifferentApplications
 
 In this directory, users can learn how to submit multiple jobs that each use a
 different molecular dynamics package. Just execute the following command on a 
 desktop:
 
-longbow -job job.conf -verbose
+longbow --job job.conf --verbose
 
 ################################################################################
 LongbowExamples/CondorSubmission

Longbow/corelibs/applications.py

@@ -74,9 +74,7 @@ def testapp(jobs):
             checked[jobs[job]["resource"]] = []
 
         # Now check if we have tested this exec already.
-        if (jobs[job]["executable"] not in checked[jobs[job]["resource"]] and
-                (jobs[job]["nochecks"] is False or
-                 jobs[job]["nochecks"] == "false")):
+        if jobs[job]["executable"] not in checked[jobs[job]["resource"]]:
 
             # If not then add it to the list now.
             checked[jobs[job]["resource"]].extend([jobs[job]["executable"]])
@@ -135,6 +133,7 @@ def processjobs(jobs):
         filelist = []
         appplugins = getattr(apps, "PLUGINEXECS")
         app = appplugins[jobs[job]["executable"]]
+        foundflags = []
         substitution = {}
 
         LOG.debug("Command-line arguments for job '%s' are '%s'",
@@ -171,14 +170,15 @@ def processjobs(jobs):
 
             substitution = getattr(
                 apps, app.lower()).detectsubstitutions(
-                    list(job["executableargs"]))
+                    list(jobs[job]["executableargs"]))
 
         except AttributeError:
 
             pass
 
         # Process the command-line.
-        foundflags = _proccommandline(jobs[job], filelist, substitution)
+        foundflags = _proccommandline(jobs[job], filelist, foundflags,
+                                      substitution)
 
         # Validate if all required flags are present.
         _flagvalidator(jobs[job], foundflags)
@@ -237,51 +237,55 @@ def _flagvalidator(job, foundflags):
             .format(job["jobname"], flags, app))
 
 
-def _proccommandline(job, filelist, substitution):
-    """Command-line processor.
+def _markfoundfiles(arg, initargs, foundflags):
+    """Method to mark file flags as found."""
+    try:
 
-    This method selects which type of command-line we have.
+        pos = initargs.index(arg) - 1
 
-    """
-    # Initialisation.
-    args = list(job["executableargs"])
-    initargs = list(job["executableargs"])
+    except ValueError:
 
-    try:
+        pos = initargs.index("../" + arg) - 1
 
-        # Determine the command-line type and call the processor method. Start
-        # with command-lines of the type exec < input.file.
-        if args[0] == "<" and len(args) > 1:
+    # In cases where there is a single input file as the first parameter. This
+    # should cover cases such as:
+    # exec input.file
+    # exec input.file > output.file
+    if arg == initargs[0]:
 
-            # Command-line type exec < input.file
-            foundflags = _procfiles(job, args[1], initargs, filelist,
-                                    substitution)
+        foundflags.append("<")
 
-        elif len(args) == 1 and args[0] != "<":
+    # All other cases should pretty much be formats like:
+    # exec -flag file -flag file -flag file
+    elif len(initargs) > 1 and initargs[pos] not in foundflags:
 
-            # Command-line type exec input.file
-            foundflags = _procfiles(job, args[0], initargs, filelist,
-                                    substitution)
+        foundflags.append(initargs[pos])
 
-        elif "-" in args[0]:
+    return foundflags
 
-            for arg in args:
 
-                # Command-line type exec -i file -c file
-                foundflags = _procfiles(job, arg, initargs, filelist,
-                                        substitution)
+def _proccommandline(job, filelist, foundflags, substitution):
+    """Command-line processor.
+
+    This method selects which type of command-line we have.
 
-        elif "-" not in args[0] and "-" in args[1]:
+    """
+    # Initialisation.
+    appplugins = getattr(apps, "PLUGINEXECS")
+    app = appplugins[job["executable"]]
+    args = list(job["executableargs"])
+    subexecs = getattr(
+        apps, app.lower()).EXECDATA[job["executable"]]["subexecutables"]
 
-            for arg in args[1:]:
+    try:
 
-                # Command-line type exec subexec -i file -c file
-                foundflags = _procfiles(job, arg, initargs, filelist,
-                                        substitution)
+        for arg in args:
 
-        else:
+            if (arg != "<" and arg != ">" and arg[0] != "-" and
+                    arg not in subexecs):
 
-            raise ValueError
+                foundflags = _procfiles(job, arg, filelist, foundflags,
+                                        substitution)
 
     except (IndexError, ValueError):
 
@@ -294,12 +298,12 @@ def _proccommandline(job, filelist, substitution):
     return foundflags
 
 
-def _procfiles(job, arg, initargs, filelist, substitution):
+def _procfiles(job, arg, filelist, foundflags, substitution):
     """Processor for finding flags and files."""
     # Initialisation.
     appplugins = getattr(apps, "PLUGINEXECS")
     app = appplugins[job["executable"]]
-    foundflags = []
+    initargs = list(job["executableargs"])
 
     # Check for as many files as there are replicates (default of 1).
     for rep in range(1, int(job["replicates"]) + 1):
@@ -327,16 +331,7 @@ def _procfiles(job, arg, initargs, filelist, substitution):
         # If we have a valid file
         if os.path.isfile(os.path.join(job["localworkdir"], fileitem)):
 
-            # Mark files as found.
-            if (len(initargs) > 1 and initargs[initargs.index(arg) - 1] not in
-                    foundflags):
-
-                foundflags.append(initargs[initargs.index(arg) - 1])
-
-            elif(len(initargs) == 1 and initargs[initargs.index(arg) - 1] not
-                    in foundflags):
-
-                foundflags.append("<")
+            _markfoundfiles(arg, initargs, foundflags)
 
             # Search input file for any file dependencies.
             try:

Longbow/corelibs/configuration.py

@@ -76,24 +76,23 @@
     "email-flags": "",
     "executable": "",
     "executableargs": "",
-    "frequency": "300",
     "handler": "",
     "host": "",
     "localworkdir": "",
     "modules": "",
     "maxtime": "24:00",
     "memory": "",
-    "nochecks": False,
-    "scripts": "",
-    "staging-frequency": "300",
-    "sge-peflag": "mpi",
-    "sge-peoverride": "false",
+    "polling-frequency": "300",
     "port": "22",
     "queue": "",
     "remoteworkdir": "",
     "resource": "",
     "replicates": "1",
     "scheduler": "",
+    "scripts": "",
+    "staging-frequency": "300",
+    "sge-peflag": "mpi",
+    "sge-peoverride": "false",
     "user": "",
     "upload-exclude": "",
     "upload-include": ""
@@ -428,22 +427,7 @@ def _processconfigsfinalinit(jobs):
 
             jobs[job]["localworkdir"] = os.getcwd()
 
-        # Fix for python 3 where basestring is now str.
-        try:
-
-            # If the exec arguments are in string form, split to list.
-            if isinstance(jobs[job]["executableargs"], basestring):
-
-                jobs[job]["executableargs"] = (
-                    jobs[job]["executableargs"].split())
-
-        except NameError:
-
-            # If the exec arguments are in string form, split to list.
-            if isinstance(jobs[job]["executableargs"], str):
-
-                jobs[job]["executableargs"] = (
-                    jobs[job]["executableargs"].split())
+        jobs[job]["executableargs"] = jobs[job]["executableargs"].split()
 
         # If modules hasn't been set then try and use a default.
         if jobs[job]["modules"] is "":
@@ -468,7 +452,7 @@ def _processconfigsparams(jobs, parameters, jobdata, hostdata):
         for item in jobs[job]:
 
             # This should already be dealt with.
-            if item is not "resource":
+            if item != "resource":
 
                 # Command-line overrides are highest priority.
                 if item in parameters and parameters[item] is not "":

Longbow/corelibs/entrypoints.py

@@ -90,7 +90,7 @@ def main():
         "debug": False,
         "disconnect": False,
         "executable": "",
-        "executableargs": [],
+        "executableargs": "",
         "hosts": "",
         "job": "",
         "jobname": "",
@@ -262,7 +262,9 @@ def longbowmain(parameters):
 
     # Test that for the applications listed in the job configuration
     # file are available and that the executable is present.
-    applications.testapp(jobs)
+    if parameters["nochecks"] is False:
+
+        applications.testapp(jobs)
 
     # Process the jobs command line arguments and find files for
     # staging.
@@ -448,7 +450,7 @@ def _commandlineproc(alllongbowargs, cmdlnargs, parameters):
                 "Recognised arguments are: {1}".format(item, allowedargs))
 
     parameters["executable"] = executable
-    parameters["executableargs"] = execargs
+    parameters["executableargs"] = " ".join(execargs)
 
     return longbowargs