diff --git a/build.gradle.kts b/build.gradle.kts index 5d09412..1a0fbd4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -13,7 +13,7 @@ plugins { } group = "io.krews" -version = "0.15.2" +version = "0.16.0" repositories { maven { setUrl("https://dl.bintray.com/kotlin/kotlin-eap") } @@ -47,6 +47,9 @@ dependencies { testImplementation("org.junit.jupiter", "junit-jupiter", "5.4.0") testImplementation("org.assertj", "assertj-core", "3.11.1") testImplementation("io.mockk", "mockk", "1.9.3") + implementation("io.ktor", "ktor-server-netty", "1.6.2") + implementation("io.ktor", "ktor-server-core", "1.6.2") + implementation("io.ktor", "ktor-html-builder", "1.6.2") } tasks.withType { diff --git a/src/main/kotlin/krews/App.kt b/src/main/kotlin/krews/App.kt index efc8a96..0bb9759 100644 --- a/src/main/kotlin/krews/App.kt +++ b/src/main/kotlin/krews/App.kt @@ -3,6 +3,7 @@ package krews import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.options.* import com.github.ajalt.clikt.parameters.types.choice +import com.github.ajalt.clikt.parameters.types.int import com.typesafe.config.ConfigFactory import krews.config.* import krews.core.* @@ -41,6 +42,8 @@ class KrewsApp(private val workflowBuilder: WorkflowBuilder) : CliktCommand() { .convert { Paths.get(it) } private val config by option("-c", "--config") .convert { Paths.get(it) } + private val httpPort by option("-p", "--http-port", help = "optional port on which to serve status reports") + .int() override fun run() { if (config != null && !Files.exists(config)) throw Exception("Given config $config not found") @@ -63,7 +66,7 @@ class KrewsApp(private val workflowBuilder: WorkflowBuilder) : CliktCommand() { val runTimestampOverride = System.getenv(WORKFLOW_RUN_TIMESTAMP_ENV_VAR)?.toLong() val taskConfigs = createTaskConfigs(hoconConfig, workflow) - val runner = WorkflowRunner(workflow, workflowConfig, taskConfigs, executor, runTimestampOverride) + val runner = WorkflowRunner(workflow, workflowConfig, taskConfigs, executor, runTimestampOverride, httpPort) // Add shutdown hook to stop all running tasks and other cleanup work if master is stopped. Runtime.getRuntime().addShutdownHook(Thread { diff --git a/src/main/kotlin/krews/core/WorkflowRunner.kt b/src/main/kotlin/krews/core/WorkflowRunner.kt index ee82d95..750bd2e 100644 --- a/src/main/kotlin/krews/core/WorkflowRunner.kt +++ b/src/main/kotlin/krews/core/WorkflowRunner.kt @@ -16,7 +16,12 @@ import java.nio.file.* import java.util.concurrent.* import java.util.concurrent.atomic.AtomicBoolean import kotlin.math.max - +import io.ktor.application.* +import io.ktor.features.ContentNegotiation +import io.ktor.http.content.* +import io.ktor.routing.* +import io.ktor.server.engine.embeddedServer +import io.ktor.server.netty.Netty private val log = KotlinLogging.logger {} @@ -25,7 +30,8 @@ class WorkflowRunner( private val workflowConfig: WorkflowConfig, private val taskConfigs: Map, private val executor: LocallyDirectedExecutor, - runTimestampOverride: Long? = null + runTimestampOverride: Long? = null, + private val httpPort: Int? = null ) { private val runRepo: RunRepo private val runDb: Database @@ -46,6 +52,8 @@ class WorkflowRunner( runRepo = RunRepo(runDb) val reportThreadFactory = BasicThreadFactory.Builder().namingPattern("report-gen-%d").build() reportPool = Executors.newSingleThreadScheduledExecutor(reportThreadFactory) + + if (httpPort != null) runHTTPServer(httpPort) } fun run() { @@ -71,6 +79,16 @@ class WorkflowRunner( onShutdown() } + private fun runHTTPServer(port: Int = 8080) { + embeddedServer(Netty, port) { + routing { + static("/") { + files("$RUN_DIR/${workflowRun.startTime}") + } + } + }.start(wait = true) + } + private fun runWorkflow(): Boolean { val taskRunner = TaskRunner(workflowRun, workflowConfig, taskConfigs, executor, runRepo) try { diff --git a/src/test/kotlin/krews/util/Workflows.kt b/src/test/kotlin/krews/util/Workflows.kt index ed207ed..e775c95 100644 --- a/src/test/kotlin/krews/util/Workflows.kt +++ b/src/test/kotlin/krews/util/Workflows.kt @@ -111,7 +111,7 @@ fun gsFilesWorkflow() = workflow("gs-files-workflow") { """ } - val gpus = task("gpus", inputFiles) { + task("gpus", inputFiles) { dockerImage = "nvidia/cuda:12.1.0-devel-ubuntu18.04" output = OutputFile("nvidia-smi/${input.filenameNoExt()}.nvidia-smi-output.txt") command =