fixed #40: allow multi-line commands to be submitted in batch mode

Changes.md

@@ -3,4 +3,5 @@
 ## Joblist v0.6
 
 
-* Implement idomatic way to output err-logs of failed jobs (fixed [#37](https://github.com/holgerbrandl/joblist/issues/37)).
+* Implement idomatic way to output err-logs of failed jobs ([#37](https://github.com/holgerbrandl/joblist/issues/37))
+* More conservative remaining runtime estimation ([#34](https://github.com/holgerbrandl/joblist/issues/34))

build.sbt

@@ -20,7 +20,7 @@ libraryDependencies += "com.offbytwo" % "docopt" % "0.6.0.20150202"
 
 // common utilities
 // git clone https://github.com/holgerbrandl/scalautils.git su_tmp && cd su_tmp && sbt publish && rm -rf ../su_tmp
-libraryDependencies += "de.mpicbg.scicomp" % "scalautils_2.11" % "0.1"
+libraryDependencies += "de.mpicbg.scicomp" % "scalautils_2.11" % "0.2-SNAPSHOT"
 //libraryDependencies += "com.github.pathikrit" %% "better-files" % "2.14.0"
 
 

docs/devel_joblist.md

@@ -30,7 +30,11 @@ sbt test:console
 ****
 ## Release Check List
 
-0) make sure that `joblist/JobListCLI.scala:99` is commented out
+Before getting started:
+* make sure that `joblist/JobListCLI.scala:99` is commented out
+* make sure to now use any non-public SNAPSHOT dependencies
+
+
 1) Increment version in build.sbt, `joblist.JobListCLI.version`
 2) Update version in README.md (installation and sbt inclusion)
 2) Push and and create version on github

faq.md

@@ -35,13 +35,25 @@ Since jl startup is limited by the underlying java VM, subsequent invocation mig
 
 There are 2 options to overcome this limitation
 
-* Batch submissions: `jl` can read job definitions from stdin. It expects one job per line
+* Batch submissions: `jl` can read job definitions from stdin (or from a file as well). By default it expects one job per line
 
 	```
 	for job_nr in $(seq 1 10); do
 	    echo "sleepTime=$(perl -e 'print int(rand(20)) + 10'); sleep $sleepTime; echo slept for $sleepTime seconds in job $job_nr"
 	done | jl submit --batch -
 	```
+	`joblist` also allows to use a custom separator charactor to process multi-line commands with this pattern
+
+    ```
+    for job_nr in $(seq 1 10); do
+        echo "
+        ## another job nr${job_nr}
+        sleepTime=$(perl -e 'print int(rand(20)) + 10');
+        sleep $sleepTime;
+        echo slept for $sleepTime seconds in job $job_nr
+        " | sed 's/^ *//' ## delete leading whitespace to allow for more robust regex
+	done | jl submit --batch - --bsep '^##'
+    ```
 
 * Loop redirection: To just monitor a large number of jobs simply [pipe](http://stackoverflow.com/questions/18612603/redirecting-output-of-bash-for-loop) your submission loop into jl
 

src/main/scala/joblist/JobListCLI.scala

@@ -30,7 +30,7 @@ object JobListCLI extends App {
       s"""
     JobList       v$version
     Description   An hpc-job manager
-    Copyright     2015 Holger Brandl
+    Copyright     2016 Holger Brandl
     License       Simplified BSD
     Website       https://github.com/holgerbrandl/joblist
     """.alignLeft.trim)
@@ -123,6 +123,7 @@ object JobListCLI extends App {
                                           execution state
      --wait                               Reset the joblist, add the job and wait until it is done. Useful for single
                                           clusterized tasks
+     --bsep <separator_pattern>           Batch separator character to separate jobs [default: ^]
       """.alignLeft.trim)
 
     val jl = getJL(options, "jl")
@@ -143,27 +144,54 @@ object JobListCLI extends App {
     val jobConfigs = ListBuffer.empty[JobConfiguration]
 
 
+    // for corresponding unit test see joblist/TestCLI.scala:113
     if (options.get("batch").get.toBoolean) {
       val batchArg = options.get("cmds_file").get
 
-      // either read lines from file or from stdin if - is used
-      val lines = if (batchArg == "-") {
-        io.Source.stdin.getLines()
+      // either read cmds from file or from stdin if - is used
+      val cmds = if (batchArg == "-") {
+
+        // fetch the command separator and split up stdin into into job commands
+        val batchSep = options.get("bsep").get.r.unanchored
+
+        // https://www.safaribooksonline.com/library/view/scala-cookbook/9781449340292/ch01s07.html
+        // "es".r.unanchored.findFirstIn("tst").isDefined
+
+        // io.Source.stdin.getLines() // old approach without customizable batch separation
+
+
+        // http://stackoverflow.com/questions/7293617/split-up-a-list-at-each-element-satisfying-a-predicate-scala
+        // http://stackoverflow.com/questions/14613995/whats-scalas-idiomatic-way-to-split-a-list-by-separator
+        File("test_data/test_stdin.txt").lines.
+          //        io.Source.stdin.getLines().
+          foldLeft(Seq(Seq.empty[String])) {
+          (acc, line) =>
+            if (batchSep.findFirstIn(line).isDefined) acc :+ Seq(line)
+            else acc.init :+ (acc.last :+ line)
+        }.map(_.mkString("\n"))
+
       } else {
         val batchFile = File(batchArg)
         require(batchFile.isRegularFile, s"batch file '${batchFile.name}' does not exist")
 
+
         batchFile.lines
       }
 
-      // convert all lines into jobs
-      lines.foreach(batchCmd => {
 
-        jobConfigs += baseConfig.copy(
-          cmd = batchCmd,
-          name = options.getOrElse("name", "jobs") + "_batch" + jobConfigs.size
-        ) // todo unit-test batch submission
-      })
+      // convert all cmds into jobs
+      cmds.
+        // remove empty elements due to formatting of input
+        filterNot(_.isEmpty).
+        map(_.alignLeft).
+        map(_.trim).
+        foreach(batchCmd => {
+
+          jobConfigs += baseConfig.copy(
+            cmd = batchCmd,
+            name = options.getOrElse("name", "jobs") + "_batch" + jobConfigs.size
+          ) // todo unit-test batch submission
+        })
 
     } else {
       jobConfigs += baseConfig.copy(

src/main/scala/joblist/package.scala

@@ -297,17 +297,17 @@ package object joblist {
     }
   }
 
-  // todo rather use scala-utils version instead once release to jcenter
-  implicit class MiscVectorUtils(values: Seq[Double]) {
-
-
-    def quantile(quantile:Double) = {
-      assert(quantile >=0 && quantile <=1)
-      // convert quantile into and index
-      val quantIndex = (values.length.toDouble*quantile).round.toInt -1
-      values.sorted.get(quantIndex)
-    }
-  }
+//  // todo rather use scala-utils version instead once release to jcenter
+//  implicit class MiscVectorUtils(values: Seq[Double]) {
+//
+//
+//    def quantile(quantile:Double) = {
+//      assert(quantile >=0 && quantile <=1)
+//      // convert quantile into and index
+//      val quantIndex = (values.length.toDouble*quantile).round.toInt -1
+//      values.sorted.get(quantIndex)
+//    }
+//  }
 
   class ListStatus(jl: JobList) {
 

src/test/scala/joblist/TestAssembledCLI.scala

@@ -61,4 +61,28 @@ class TestAssembledCLI extends FlatSpec with Matchers with BeforeAndAfter {
     result.exitCode should be(0)
     File("script_result.txt").toJava should exist
   }
+
+
+
+
+  it should "split batch jobs up correctly" in {
+
+    val script = s"""
+    cd ${wd.pathAsString}
+
+    jl reset
+    cat ${home/"test_data"/"test_stdin.txt"} | jl --batch - --bsep '^##'
+
+    jl wait
+    """.alignLeft
+
+    println(script)
+
+    Bash.eval(script, showOutput = true)
+
+    val jl = JobList()
+
+    jl.file.withExt(".html").toJava should exist
+    jl.jobs.size should be (3)
+  }
 }

test_data/test_stdin.txt

@@ -0,0 +1,16 @@
+## another job nr1
+sleepTime=12;
+sleep ;
+echo slept for  seconds in job 1
+
+
+## another job nr2
+sleepTime=28;
+sleep ;
+echo slept for  seconds in job 2
+
+
+## another job nr3
+sleepTime=10;
+sleep ;
+echo slept for  seconds in job 3