allow serving status reports by HTTP

build.gradle.kts

@@ -13,7 +13,7 @@ plugins {
 }
 
 group = "io.krews"
-version = "0.15.2"
+version = "0.16.0"
 
 repositories {
     maven { setUrl("https://dl.bintray.com/kotlin/kotlin-eap") }
@@ -47,6 +47,9 @@ dependencies {
     testImplementation("org.junit.jupiter", "junit-jupiter", "5.4.0")
     testImplementation("org.assertj", "assertj-core", "3.11.1")
     testImplementation("io.mockk", "mockk", "1.9.3")
+    implementation("io.ktor", "ktor-server-netty", "1.6.2")
+    implementation("io.ktor", "ktor-server-core", "1.6.2")
+    implementation("io.ktor", "ktor-html-builder", "1.6.2")
 }
 
 tasks.withType<Test> {

src/main/kotlin/krews/App.kt

@@ -3,6 +3,7 @@ package krews
 import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.parameters.options.*
 import com.github.ajalt.clikt.parameters.types.choice
+import com.github.ajalt.clikt.parameters.types.int
 import com.typesafe.config.ConfigFactory
 import krews.config.*
 import krews.core.*
@@ -41,6 +42,8 @@ class KrewsApp(private val workflowBuilder: WorkflowBuilder) : CliktCommand() {
         .convert { Paths.get(it) }
     private val config by option("-c", "--config")
         .convert { Paths.get(it) }
+    private val httpPort by option("-p", "--http-port", help = "optional port on which to serve status reports")
+        .int()
 
     override fun run() {
         if (config != null && !Files.exists(config)) throw Exception("Given config $config not found")
@@ -63,7 +66,7 @@ class KrewsApp(private val workflowBuilder: WorkflowBuilder) : CliktCommand() {
 
             val runTimestampOverride = System.getenv(WORKFLOW_RUN_TIMESTAMP_ENV_VAR)?.toLong()
             val taskConfigs = createTaskConfigs(hoconConfig, workflow)
-            val runner = WorkflowRunner(workflow, workflowConfig, taskConfigs, executor, runTimestampOverride)
+            val runner = WorkflowRunner(workflow, workflowConfig, taskConfigs, executor, runTimestampOverride, httpPort)
 
             // Add shutdown hook to stop all running tasks and other cleanup work if master is stopped.
             Runtime.getRuntime().addShutdownHook(Thread {

src/main/kotlin/krews/core/WorkflowRunner.kt

@@ -16,7 +16,12 @@ import java.nio.file.*
 import java.util.concurrent.*
 import java.util.concurrent.atomic.AtomicBoolean
 import kotlin.math.max
-
+import io.ktor.application.*
+import io.ktor.features.ContentNegotiation
+import io.ktor.http.content.*
+import io.ktor.routing.*
+import io.ktor.server.engine.embeddedServer
+import io.ktor.server.netty.Netty
 
 private val log = KotlinLogging.logger {}
 
@@ -25,7 +30,8 @@ class WorkflowRunner(
     private val workflowConfig: WorkflowConfig,
     private val taskConfigs: Map<String, TaskConfig>,
     private val executor: LocallyDirectedExecutor,
-    runTimestampOverride: Long? = null
+    runTimestampOverride: Long? = null,
+    private val httpPort: Int? = null
 ) {
     private val runRepo: RunRepo
     private val runDb: Database
@@ -46,6 +52,8 @@ class WorkflowRunner(
         runRepo = RunRepo(runDb)
         val reportThreadFactory = BasicThreadFactory.Builder().namingPattern("report-gen-%d").build()
         reportPool = Executors.newSingleThreadScheduledExecutor(reportThreadFactory)
+
+        if (httpPort != null) runHTTPServer(httpPort)
     }
 
     fun run() {
@@ -71,6 +79,16 @@ class WorkflowRunner(
         onShutdown()
     }
 
+    private fun runHTTPServer(port: Int = 8080) {
+        embeddedServer(Netty, port) {
+            routing {
+                static("/") {
+                    files("$RUN_DIR/${workflowRun.startTime}")
+                }
+            }
+        }.start(wait = true)
+    }
+
     private fun runWorkflow(): Boolean {
         val taskRunner = TaskRunner(workflowRun, workflowConfig, taskConfigs, executor, runRepo)
         try {

src/test/kotlin/krews/util/Workflows.kt

@@ -111,7 +111,7 @@ fun gsFilesWorkflow() = workflow("gs-files-workflow") {
             """
     }
 
-    val gpus = task<File, OutputFile>("gpus", inputFiles) {
+    task<File, OutputFile>("gpus", inputFiles) {
         dockerImage = "nvidia/cuda:12.1.0-devel-ubuntu18.04"
         output = OutputFile("nvidia-smi/${input.filenameNoExt()}.nvidia-smi-output.txt")
         command =