From ac228313d9246f19d09a023dc339dd01105a9d10 Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Mon, 27 Jul 2020 16:20:57 +0300 Subject: [PATCH 1/7] Refactor libraries handling, add handling of descriptors from different sources. Fixes #70 Also, get rid of several singletons in config and replaced them with factory and free functions. --- README.md | 20 ++ .../kotlin/jupyter/annotationsProcessor.kt | 4 +- .../org/jetbrains/kotlin/jupyter/commands.kt | 26 +- .../org/jetbrains/kotlin/jupyter/config.kt | 274 +++--------------- .../org/jetbrains/kotlin/jupyter/ikotlin.kt | 14 +- .../LibrariesProcessor.kt} | 25 +- .../jupyter/libraries/LibraryCacheable.kt | 6 + .../jupyter/libraries/LibraryFactory.kt | 63 ++++ .../jupyter/libraries/LibraryReference.kt | 28 ++ .../libraries/LibraryResolutionInfo.kt | 119 ++++++++ .../libraries/LibraryResolutionInfoParser.kt | 60 ++++ .../jupyter/libraries/LibraryResolver.kt | 80 +++++ .../kotlin/jupyter/libraries/util.kt | 136 +++++++++ .../org/jetbrains/kotlin/jupyter/magics.kt | 25 +- .../org/jetbrains/kotlin/jupyter/message.kt | 2 +- .../org/jetbrains/kotlin/jupyter/protocol.kt | 42 ++- .../org/jetbrains/kotlin/jupyter/repl.kt | 100 +++++-- .../jupyter/repl/{spark => }/ClassWriter.kt | 2 +- .../repl/{reflect => }/ContextUpdater.kt | 2 +- .../repl/{completion => }/KotlinCompleter.kt | 6 +- .../jetbrains/kotlin/jupyter/typeProviders.kt | 4 +- .../org/jetbrains/kotlin/jupyter/util.kt | 26 +- .../kotlin/jupyter/test/configTests.kt | 14 +- .../kotlin/jupyter/test/ikotlinTests.kt | 4 +- .../jupyter/test/kernelServerTestsBase.kt | 7 +- .../kotlin/jupyter/test/parseMagicsTests.kt | 70 ++++- .../kotlin/jupyter/test/replTests.kt | 134 +++++++-- .../jetbrains/kotlin/jupyter/test/testUtil.kt | 70 ++++- .../kotlin/jupyter/test/typeProviderTests.kt | 12 +- 29 files changed, 981 insertions(+), 394 deletions(-) rename src/main/kotlin/org/jetbrains/kotlin/jupyter/{libraries.kt => libraries/LibrariesProcessor.kt} (85%) create mode 100644 src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryCacheable.kt create mode 100644 src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryFactory.kt create mode 100644 src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryReference.kt create mode 100644 src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolutionInfo.kt create mode 100644 src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolutionInfoParser.kt create mode 100644 src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolver.kt create mode 100644 src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/util.kt rename src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/{spark => }/ClassWriter.kt (97%) rename src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/{reflect => }/ContextUpdater.kt (98%) rename src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/{completion => }/KotlinCompleter.kt (95%) diff --git a/README.md b/README.md index ff856155a..f9dd9edfd 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ The following line magics are supported: - `%use , ...` - injects code for supported libraries: artifact resolution, default imports, initialization code, type renderers - `%trackClasspath` - logs any changes of current classpath. Useful for debugging artifact resolution failures - `%trackExecution` - logs pieces of code that are going to be executed. Useful for debugging of libraries support + - `%useLatestDescriptors` - use latest versions of library descriptors available. By default, bundled descriptors are used - `%output [options]` - output capturing settings. See detailed info about line magics [here](doc/magics.md). @@ -128,6 +129,25 @@ Several libraries can be included in single `%use` statement, separated by `,`: ``` %use lets-plot, krangl, mysql(8.0.15) ``` +You can also specify the source of library descriptor. By default, it's downloaded from the latest commit on the +branch which kernel was built from. If you want to try descriptor from another revision, use the following syntax: +``` +// Specify tag +%use lets-plot@0.8.2.5 +// Specify commit sha, with more verbose syntax +%use lets-plot@ref[24a040fe22335648885b106e2f4ddd63b4d49469] +// Specify git ref along with library arguments +%use krangl@dev(0.10) +``` +Other options are resolving library descriptor from a local file or from remote URL: +``` +// Load library from file +%use mylib@file[/home/user/lib.json] +// Load library descriptor from a remote URL +%use herlib@url[https://site.com/lib.json] +// You may omit library name for file and URL resolution: +%use @file[lib.json] +``` List of supported libraries: - [klaxon](https://github.com/cbeust/klaxon) - JSON parser for Kotlin diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt index fdf6f64c5..60f0eb553 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt @@ -1,7 +1,7 @@ package org.jetbrains.kotlin.jupyter import jupyter.kotlin.KotlinFunctionInfo -import org.jetbrains.kotlin.jupyter.repl.reflect.ContextUpdater +import org.jetbrains.kotlin.jupyter.repl.ContextUpdater interface AnnotationsProcessor { @@ -57,4 +57,4 @@ class AnnotationsProcessorImpl(private val contextUpdater: ContextUpdater) : Ann } return codeToExecute } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt index 6c57c6981..066639411 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt @@ -1,10 +1,11 @@ package org.jetbrains.kotlin.jupyter import jupyter.kotlin.textResult -import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResult -import org.jetbrains.kotlin.jupyter.repl.completion.KotlinCompleter -import org.jetbrains.kotlin.jupyter.repl.completion.ListErrorsResult -import org.jetbrains.kotlin.jupyter.repl.completion.SourceCodeImpl +import org.jetbrains.kotlin.jupyter.libraries.parseLibraryDescriptor +import org.jetbrains.kotlin.jupyter.repl.CompletionResult +import org.jetbrains.kotlin.jupyter.repl.KotlinCompleter +import org.jetbrains.kotlin.jupyter.repl.ListErrorsResult +import org.jetbrains.kotlin.jupyter.repl.SourceCodeImpl import kotlin.script.experimental.api.ScriptDiagnostic import kotlin.script.experimental.api.SourceCode import kotlin.script.experimental.api.SourceCodeCompletionVariant @@ -51,12 +52,12 @@ fun doCommandCompletion(code: String, cursor: Int): CompletionResult { return KotlinCompleter.getResult(code, cursor, completions) } -fun runCommand(code: String, repl: ReplForJupyter?): Response { +fun runCommand(code: String, repl: ReplForJupyter): Response { val args = code.trim().substring(1).split(" ") val cmd = getCommand(args[0]) ?: return AbortResponseWithMessage(textResult("Failed!"), "unknown command: $code\nto see available commands, enter :help") return when (cmd) { ReplCommands.classpath -> { - val cp = repl!!.currentClasspath + val cp = repl.currentClasspath OkResponseWithMessage(textResult("Current classpath (${cp.count()} paths):\n${cp.joinToString("\n")}")) } ReplCommands.help -> { @@ -66,9 +67,16 @@ fun runCommand(code: String, repl: ReplForJupyter?): Response { if (it.argumentsUsage != null) s += "\n Usage: %${it.name} ${it.argumentsUsage}" s } - val libraries = repl?.resolverConfig?.libraries?.awaitBlocking()?.toList()?.joinToStringIndented { - "${it.first} ${it.second.link ?: ""}" - } + val libraryFiles = + repl.homeDir?.resolve(LibrariesDir)?.listFiles { file -> file.isFile && file.name.endsWith(".$LibraryDescriptorExt") } ?: emptyArray() + val libraries = libraryFiles.toList().mapNotNull { file -> + val libraryName = file.nameWithoutExtension + log.info("Parsing descriptor for library '$libraryName'") + val descriptor = log.catchAll("Parsing descriptor for library '$libraryName' failed") { + parseLibraryDescriptor(file.readText()) + } + if (descriptor != null) "$libraryName ${descriptor.link ?: ""}" else null + }.joinToStringIndented() OkResponseWithMessage(textResult("Commands:\n$commands\n\nMagics\n$magics\n\nSupported libraries:\n$libraries")) } } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt index a5dafb967..ec1363e51 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt @@ -5,34 +5,45 @@ import com.beust.klaxon.Parser import jupyter.kotlin.JavaRuntime import jupyter.kotlin.KotlinKernelVersion import khttp.responses.Response -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.async -import org.apache.commons.io.FileUtils -import org.json.JSONObject +import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory +import org.jetbrains.kotlin.jupyter.libraries.LibraryResolver import org.slf4j.LoggerFactory import org.zeromq.SocketType import java.io.File -import java.nio.file.Files import java.nio.file.Paths import kotlin.script.experimental.dependencies.RepositoryCoordinates -val LibrariesDir = "libraries" -val LocalCacheDir = "cache" -val CachedLibrariesFootprintFile = "libsCommit" +const val LibrariesDir = "libraries" +const val LocalCacheDir = "cache" val LocalSettingsPath = Paths.get(System.getProperty("user.home"), ".jupyter_kotlin").toString() -val GitHubApiHost = "api.github.com" -val GitHubRepoOwner = "kotlin" -val GitHubRepoName = "kotlin-jupyter" -val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName" +const val GitHubApiHost = "api.github.com" +const val GitHubRepoOwner = "kotlin" +const val GitHubRepoName = "kotlin-jupyter" +const val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName" -val LibraryDescriptorExt = "json" -val LibraryPropertiesFile = ".properties" +const val LibraryDescriptorExt = "json" +const val LibraryPropertiesFile = ".properties" + +const val protocolVersion = "5.3" internal val log by lazy { LoggerFactory.getLogger("ikotlin") } +val defaultRuntimeProperties by lazy { + RuntimeKernelProperties(ClassLoader.getSystemResource("runtime.properties")?.readText()?.parseIniConfig().orEmpty()) +} + +val defaultRepositories = arrayOf( + "https://jcenter.bintray.com/", + "https://repo.maven.apache.org/maven2/", + "https://jitpack.io", +).map { RepositoryCoordinates(it) } + +val defaultGlobalImports = listOf( + "kotlin.math.*", +) + enum class JupyterSockets(val zmqKernelType: SocketType, val zmqClientType: SocketType) { hb(SocketType.REP, SocketType.REQ), shell(SocketType.ROUTER, SocketType.REQ), @@ -57,25 +68,21 @@ data class OutputConfig( } } -data class RuntimeKernelProperties(val map: Map) { - val version: KotlinKernelVersion? by lazy { +class RuntimeKernelProperties(val map: Map): ReplRuntimeProperties { + override val version: KotlinKernelVersion? by lazy { map["version"]?.let{ KotlinKernelVersion.from(it) } } - val librariesFormatVersion: Int + override val librariesFormatVersion: Int get() = map["librariesFormatVersion"]?.toIntOrNull() ?: throw RuntimeException("Libraries format version is not specified!") - val currentBranch: String + override val currentBranch: String get() = map["currentBranch"] ?: throw RuntimeException("Current branch is not specified!") - val currentSha: String + override val currentSha: String get() = map["currentSha"] ?: throw RuntimeException("Current commit SHA is not specified!") - val jvmTargetForSnippets by lazy { + override val jvmTargetForSnippets by lazy { map["jvmTargetForSnippets"] ?: JavaRuntime.version } } -val runtimeProperties by lazy { - RuntimeKernelProperties(ClassLoader.getSystemResource("runtime.properties")?.readText()?.parseIniConfig().orEmpty()) -} - data class KernelConfig( val ports: List, val transport: String, @@ -83,9 +90,11 @@ data class KernelConfig( val signatureKey: String, val pollingIntervalMillis: Long = 100, val scriptClasspath: List = emptyList(), - val resolverConfig: ResolverConfig? + val homeDir: File?, + val resolverConfig: ResolverConfig?, + val libraryFactory: LibraryFactory, ) { - fun toArgs(prefix: String = "", homeDir: File? = null): KernelArgs { + fun toArgs(prefix: String = ""): KernelArgs { val cfgJson = jsonObject( "transport" to transport, "signature_scheme" to signatureScheme, @@ -102,7 +111,7 @@ data class KernelConfig( } companion object { - fun fromArgs(args: KernelArgs): KernelConfig { + fun fromArgs(args: KernelArgs, libraryFactory: LibraryFactory): KernelConfig { val (cfgFile, scriptClasspath, homeDir) = args val cfgJson = Parser.default().parse(cfgFile.canonicalPath) as JsonObject fun JsonObject.getInt(field: String): Int = int(field) ?: throw RuntimeException("Cannot find $field in $cfgFile") @@ -116,17 +125,17 @@ data class KernelConfig( signatureScheme = sigScheme ?: "hmac1-sha256", signatureKey = if (sigScheme == null || key == null) "" else key, scriptClasspath = scriptClasspath, - resolverConfig = homeDir?.let { loadResolverConfig(it.toString()) } + homeDir = homeDir, + resolverConfig = homeDir?.let { loadResolverConfig(it.toString(), libraryFactory) }, + libraryFactory = libraryFactory, ) } } } -val protocolVersion = "5.3" - data class TypeHandler(val className: TypeName, val code: Code) -data class Variable(val name: String, val value: String) +data class Variable(val name: String, val value: String, val required: Boolean = false) open class LibraryDefinition( val dependencies: List, @@ -141,6 +150,7 @@ open class LibraryDefinition( ) class LibraryDescriptor( + val originalJson: JsonObject, dependencies: List, val variables: List, initCell: List, @@ -156,56 +166,7 @@ class LibraryDescriptor( ) : LibraryDefinition(dependencies, initCell, imports, repositories, init, shutdown, renderers, converters, annotations) data class ResolverConfig(val repositories: List, - val libraries: Deferred>) - -fun parseLibraryArgument(str: String): Variable { - val eq = str.indexOf('=') - return if (eq == -1) Variable("", str.trim()) - else Variable(str.substring(0, eq).trim(), str.substring(eq + 1).trim()) -} - -fun parseLibraryName(str: String): Pair> { - val brackets = str.indexOf('(') - if (brackets == -1) return str.trim() to emptyList() - val name = str.substring(0, brackets).trim() - val args = str.substring(brackets + 1, str.indexOf(')', brackets)) - .split(',') - .map(::parseLibraryArgument) - return name to args -} - -fun readLibraries(basePath: String? = null, filter: (File) -> Boolean = { true }): List> { - val parser = Parser.default() - return File(basePath, LibrariesDir) - .listFiles()?.filter { it.extension == LibraryDescriptorExt && filter(it) } - ?.map { - log.info("Loading '${it.nameWithoutExtension}' descriptor from '${it.canonicalPath}'") - it.nameWithoutExtension to parser.parse(it.canonicalPath) as JsonObject - } - .orEmpty() -} - -fun getLatestCommitToLibraries(sinceTimestamp: String?): Pair? = - log.catchAll { - var url = "$GitHubApiPrefix/commits?path=$LibrariesDir&sha=${runtimeProperties.currentBranch}" - if (sinceTimestamp != null) - url += "&since=$sinceTimestamp" - log.info("Checking for new commits to library descriptors at $url") - val arr = getHttp(url).jsonArray - if (arr.length() == 0) { - if (sinceTimestamp != null) - getLatestCommitToLibraries(null) - else { - log.info("Didn't find any commits to '$LibrariesDir' at $url") - null - } - } else { - val commit = arr[0] as JSONObject - val sha = commit["sha"] as String - val timestamp = ((commit["commit"] as JSONObject)["committer"] as JSONObject)["date"] as String - sha to timestamp - } - } + val libraries: LibraryResolver) fun getHttp(url: String): Response { val response = khttp.get(url) @@ -214,151 +175,4 @@ fun getHttp(url: String): Response { return response } -fun getLibraryDescriptorVersion(commitSha: String) = - log.catchAll { - val url = "$GitHubApiPrefix/contents/$LibrariesDir/$LibraryPropertiesFile?ref=$commitSha" - log.info("Checking current library descriptor format version from $url") - val response = getHttp(url) - val downloadUrl = response.jsonObject["download_url"].toString() - val downloadResult = getHttp(downloadUrl) - val result = downloadResult.text.parseIniConfig()["formatVersion"]!!.toInt() - log.info("Current library descriptor format version: $result") - result - } - -/*** - * Downloads library descriptors from GitHub to local cache if new commits in `libraries` directory were detected - */ -fun downloadNewLibraryDescriptors() { - - // Read commit hash and timestamp for locally cached libraries. - // Timestamp is used as parameter for commits request to reduce output - - val footprintFilePath = Paths.get(LocalSettingsPath, LocalCacheDir, CachedLibrariesFootprintFile).toString() - log.info("Reading commit info for which library descriptors were cached: '$footprintFilePath'") - val footprintFile = File(footprintFilePath) - val footprint = footprintFile.tryReadIniConfig() - val timestampRegex = """\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z""".toRegex() - val syncedCommitTimestamp = footprint?.get("timestamp")?.validOrNull { timestampRegex.matches(it) } - val syncedCommitSha = footprint?.get("sha") - log.info("Local libraries are cached for commit '$syncedCommitSha' at '$syncedCommitTimestamp'") - - val (latestCommitSha, latestCommitTimestamp) = getLatestCommitToLibraries(syncedCommitTimestamp) ?: return - if (latestCommitSha.equals(syncedCommitSha)) { - log.info("No new commits to library descriptors were detected") - return - } - - // Download library descriptor version - - val descriptorVersion = getLibraryDescriptorVersion(latestCommitSha) ?: return - val libraryDescriptorFormatVersion = runtimeProperties.librariesFormatVersion - - if (descriptorVersion != libraryDescriptorFormatVersion) { - if (descriptorVersion < libraryDescriptorFormatVersion) - log.error("Incorrect library descriptor version in GitHub repository: $descriptorVersion") - else - log.warn("Kotlin Kernel needs to be updated to the latest version. Couldn't download new library descriptors from GitHub repository because their format was changed") - return - } - - // Download library descriptors - - log.info("New commits to library descriptors were detected. Downloading library descriptors for commit $latestCommitSha") - - val libraries = log.catchAll { - val url = "$GitHubApiPrefix/contents/$LibrariesDir?ref=$latestCommitSha" - log.info("Requesting the list of library descriptors at $url") - val response = getHttp(url) - val filenameRegex = """[\w.-]+\.$LibraryDescriptorExt""".toRegex() - - response.jsonArray.mapNotNull { - val o = it as JSONObject - val filename = o["name"] as String - if (filenameRegex.matches(filename)) { - val libUrl = o["download_url"].toString() - log.info("Downloading '$filename' from $libUrl") - val res = getHttp(libUrl) - val text = res.jsonObject.toString() - filename to text - } else null - } - } ?: return - - // Save library descriptors to local cache - - val librariesPath = Paths.get(LocalSettingsPath, LocalCacheDir, LibrariesDir) - val librariesDir = librariesPath.toFile() - log.info("Saving ${libraries.count()} library descriptors to local cache at '$librariesPath'") - try { - FileUtils.deleteDirectory(librariesDir) - Files.createDirectories(librariesPath) - libraries.forEach { - File(librariesDir.toString(), it.first).writeText(it.second) - } - footprintFile.writeText(""" - timestamp=$latestCommitTimestamp - sha=$latestCommitSha - """.trimIndent()) - } catch (e: Exception) { - log.error("Failed to write downloaded library descriptors to local cache:", e) - log.catchAll { FileUtils.deleteDirectory(librariesDir) } - } -} - -fun getLibrariesJsons(homeDir: String): Map { - - downloadNewLibraryDescriptors() - - val pathsToCheck = arrayOf(LocalSettingsPath, - Paths.get(LocalSettingsPath, LocalCacheDir).toString(), - homeDir) - - val librariesMap = mutableMapOf() - - pathsToCheck.forEach { - readLibraries(it) { !librariesMap.containsKey(it.nameWithoutExtension) } - .forEach { librariesMap.put(it.first, it.second) } - } - - return librariesMap -} - -fun loadResolverConfig(homeDir: String) = ResolverConfig(defaultRepositories, GlobalScope.async { - log.catchAll { - parserLibraryDescriptors(getLibrariesJsons(homeDir)) - } ?: emptyMap() -}) - -val defaultRepositories = arrayOf( - "https://jcenter.bintray.com/", - "https://repo.maven.apache.org/maven2/", - "https://jitpack.io", -).map { RepositoryCoordinates(it) } - -val defaultGlobalImports = listOf( - "kotlin.math.*", -) - -fun parserLibraryDescriptors(libJsons: Map): Map { - return libJsons.mapValues { - log.info("Parsing '${it.key}' descriptor") - LibraryDescriptor( - dependencies = it.value.array("dependencies")?.toList().orEmpty(), - variables = it.value.obj("properties")?.map { Variable(it.key, it.value.toString()) }.orEmpty(), - imports = it.value.array("imports")?.toList().orEmpty(), - repositories = it.value.array("repositories")?.toList().orEmpty(), - init = it.value.array("init")?.toList().orEmpty(), - shutdown = it.value.array("shutdown")?.toList().orEmpty(), - initCell = it.value.array("initCell")?.toList().orEmpty(), - renderers = it.value.obj("renderers")?.map { - TypeHandler(it.key, it.value.toString()) - }?.toList().orEmpty(), - link = it.value.string("link"), - minKernelVersion = it.value.string("minKernelVersion"), - converters = it.value.obj("typeConverters")?.map { TypeHandler(it.key, it.value.toString()) }.orEmpty(), - annotations = it.value.obj("annotationHandlers")?.map { TypeHandler(it.key, it.value.toString()) }.orEmpty() - ) - } -} - +fun loadResolverConfig(homeDir: String, libraryFactory: LibraryFactory) = ResolverConfig(defaultRepositories, libraryFactory.getStandardResolver(homeDir)) diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt index 56128e40d..a757912ac 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt @@ -1,5 +1,6 @@ package org.jetbrains.kotlin.jupyter +import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory import java.io.File import java.util.concurrent.atomic.AtomicLong import kotlin.concurrent.thread @@ -65,14 +66,17 @@ fun main(vararg args: String) { try { log.info("Kernel args: "+ args.joinToString { it }) val kernelArgs = parseCommandLine(*args) - val kernelConfig = KernelConfig.fromArgs(kernelArgs) - kernelServer(kernelConfig) + val runtimeProperties = defaultRuntimeProperties + val libraryPath = (kernelArgs.homeDir ?: File("")).resolve(LibrariesDir) + val libraryFactory = LibraryFactory.withDefaultDirectoryResolution(libraryPath) + val kernelConfig = KernelConfig.fromArgs(kernelArgs, libraryFactory) + kernelServer(kernelConfig, runtimeProperties) } catch (e: Exception) { log.error("exception running kernel with args: \"${args.joinToString()}\"", e) } } -fun kernelServer(config: KernelConfig) { +fun kernelServer(config: KernelConfig, runtimeProperties: ReplRuntimeProperties) { log.info("Starting server with config: $config") JupyterConnection(config).use { conn -> @@ -83,7 +87,7 @@ fun kernelServer(config: KernelConfig) { val executionCount = AtomicLong(1) - val repl = ReplForJupyterImpl(config.scriptClasspath, config.resolverConfig) + val repl = ReplForJupyterImpl(config, runtimeProperties) val mainThread = Thread.currentThread() @@ -91,7 +95,7 @@ fun kernelServer(config: KernelConfig) { while (true) { try { conn.heartbeat.onData { send(it, 0) } - conn.control.onMessage { shellMessagesHandler(it, null, executionCount) } + conn.control.onMessage { controlMessagesHandler(it, repl) } Thread.sleep(config.pollingIntervalMillis) } catch (e: InterruptedException) { diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibrariesProcessor.kt similarity index 85% rename from src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt rename to src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibrariesProcessor.kt index 2eb84e01c..4e7cb08c9 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibrariesProcessor.kt @@ -1,9 +1,18 @@ -package org.jetbrains.kotlin.jupyter +package org.jetbrains.kotlin.jupyter.libraries import jupyter.kotlin.KotlinKernelVersion -import kotlinx.coroutines.Deferred +import org.jetbrains.kotlin.jupyter.LibraryDefinition +import org.jetbrains.kotlin.jupyter.LibraryDescriptor +import org.jetbrains.kotlin.jupyter.ReplCompilerException +import org.jetbrains.kotlin.jupyter.ReplRuntimeProperties +import org.jetbrains.kotlin.jupyter.TypeHandler +import org.jetbrains.kotlin.jupyter.Variable -class LibrariesProcessor(private val libraries: Deferred>?) { +class LibrariesProcessor( + private val libraries: LibraryResolver?, + private val runtimeProperties: ReplRuntimeProperties, + val libraryFactory: LibraryFactory, +) { /** * Matches a list of actual library arguments with declared library parameters @@ -98,13 +107,13 @@ class LibrariesProcessor(private val libraries: Deferred = defaultParsers, +) { + fun parseReferenceWithArgs(str: String): Pair> { + val (fullName, vars) = parseLibraryName(str) + val reference = parseReference(fullName) + return reference to vars + } + + fun getStandardResolver(homeDir: String): LibraryResolver { + // Standard resolver doesn't cache results in memory + var res: LibraryResolver = FallbackLibraryResolver() + res = LocalLibraryResolver(res, Paths.get(homeDir, LibrariesDir).toString()) + return res + } + + private fun parseResolutionInfo(string: String): LibraryResolutionInfo { + // In case of empty string after `@`: %use lib@ + if(string.isBlank()) return defaultResolutionInfo + + val (type, vars) = parseCall(string, Brackets.SQUARE) + val parser = parsers[type] ?: return LibraryResolutionInfo.getInfoByRef(type) + return parser.getInfo(vars) + } + + private fun parseReference(string: String): LibraryReference { + val sepIndex = string.indexOf('@') + if (sepIndex == -1) return LibraryReference(defaultResolutionInfo, string) + + val nameString = string.substring(0, sepIndex) + val infoString = string.substring(sepIndex + 1) + val info = parseResolutionInfo(infoString) + return LibraryReference(info, nameString) + } + + companion object { + fun withDefaultDirectoryResolution(dir: File) = LibraryFactory(LibraryResolutionInfo.ByDir(dir)) + + private val defaultParsers = listOf( + LibraryResolutionInfoParser.make("ref", listOf(Parameter.Required("ref"))) { args -> + LibraryResolutionInfo.getInfoByRef(args["ref"] ?: error("Argument 'ref' should be specified")) + }, + LibraryResolutionInfoParser.make("file", listOf(Parameter.Required("path"))) { args -> + LibraryResolutionInfo.ByFile(File(args["path"] ?: error("Argument 'path' should be specified"))) + }, + LibraryResolutionInfoParser.make("dir", listOf(Parameter.Required("dir"))) { args -> + LibraryResolutionInfo.ByDir(File(args["dir"] ?: error("Argument 'dir' should be specified"))) + }, + LibraryResolutionInfoParser.make("url", listOf(Parameter.Required("url"))) { args -> + LibraryResolutionInfo.ByURL(URL(args["url"] ?: error("Argument 'url' should be specified"))) + }, + ).map { it.name to it }.toMap() + } +} diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryReference.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryReference.kt new file mode 100644 index 000000000..9ce8c0db7 --- /dev/null +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryReference.kt @@ -0,0 +1,28 @@ +package org.jetbrains.kotlin.jupyter.libraries + +import org.jetbrains.kotlin.jupyter.LibraryDescriptor + +data class LibraryReference( + val info: LibraryResolutionInfo, + val name: String? = null, +): LibraryCacheable by info { + + val key: String + + init { + val namePart = if (name.isNullOrEmpty()) "" else "${name}_" + key = namePart + info.key + } + + fun resolve(): LibraryDescriptor { + val text = info.resolve(name) + return parseLibraryDescriptor(text) + } + + override fun toString(): String { + val namePart = name ?: "" + val infoPart = info.toString() + return if (infoPart.isEmpty()) namePart + else "$namePart@$infoPart" + } +} diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolutionInfo.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolutionInfo.kt new file mode 100644 index 000000000..a2e197c08 --- /dev/null +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolutionInfo.kt @@ -0,0 +1,119 @@ +package org.jetbrains.kotlin.jupyter.libraries + +import org.jetbrains.kotlin.jupyter.GitHubApiPrefix +import org.jetbrains.kotlin.jupyter.LibrariesDir +import org.jetbrains.kotlin.jupyter.LibraryDescriptorExt +import org.jetbrains.kotlin.jupyter.ReplCompilerException +import org.jetbrains.kotlin.jupyter.Variable +import org.jetbrains.kotlin.jupyter.getHttp +import org.jetbrains.kotlin.jupyter.log +import java.io.File +import java.net.URL +import java.util.concurrent.ConcurrentHashMap + +abstract class LibraryResolutionInfo( + private val typeKey: String +): LibraryCacheable { + class ByNothing : LibraryResolutionInfo("nothing") { + override val args: List = listOf() + + override fun resolve(name: String?): String = "{}" + } + + class ByURL(val url: URL): LibraryResolutionInfo("url") { + override val args = listOf(Variable("url", url.toString())) + override val shouldBeCachedLocally get() = false + + override fun resolve(name: String?): String { + val response = getHttp(url.toString()) + return response.text + } + } + + class ByFile(val file: File): LibraryResolutionInfo("file") { + override val args = listOf(Variable("file", file.path)) + override val shouldBeCachedLocally get() = false + + override fun resolve(name: String?): String { + return file.readText() + } + } + + class ByDir(private val librariesDir: File): LibraryResolutionInfo("bundled") { + override val args = listOf(Variable("dir", librariesDir.path)) + override val shouldBeCachedLocally get() = false + + override fun resolve(name: String?): String { + if (name == null) throw ReplCompilerException("Directory library resolver needs library name to be specified") + + return librariesDir.resolve("$name.$LibraryDescriptorExt").readText() + } + } + + class ByGitRef(private val ref: String): LibraryResolutionInfo("ref") { + override val valueKey: String + get() = sha + + val sha: String by lazy { + val (resolvedSha, _) = getLatestCommitToLibraries(ref, null) ?: return@lazy ref + resolvedSha + } + + override val args = listOf(Variable("ref", ref)) + + override fun resolve(name: String?): String { + if (name == null) throw ReplCompilerException("Reference library resolver needs name to be specified") + + val url = "$GitHubApiPrefix/contents/$LibrariesDir/$name.$LibraryDescriptorExt?ref=$sha" + log.info("Requesting library descriptor at $url") + val response = getHttp(url).jsonObject + + val downloadURL = response["download_url"].toString() + val res = getHttp(downloadURL) + val text = res.jsonObject + return text.toString() + } + } + + protected abstract val args: List + protected open val valueKey: String + get() = args.joinToString { it.value } + + val key: String by lazy { "${typeKey}_${replaceForbiddenChars(valueKey)}" } + abstract fun resolve(name: String?): String + + override fun hashCode(): Int { + return key.hashCode() + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as LibraryResolutionInfo + + return valueKey == other.valueKey + } + + override fun toString(): String { + return typeKey + + when { + args.isEmpty() -> "" + args.size == 1 -> "[${args[0].value}]" + else -> args.joinToString(", ", "[", "]") { "${it.name}=${it.value}" } + } + } + + companion object { + private val gitRefsCache = ConcurrentHashMap() + + fun getInfoByRef(ref: String): ByGitRef { + return gitRefsCache.getOrPut(ref, { ByGitRef(ref) }) + } + + fun replaceForbiddenChars(string: String): String { + return string.replace("""[<>/\\:"|?*]""".toRegex(), "_") + } + } + +} diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolutionInfoParser.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolutionInfoParser.kt new file mode 100644 index 000000000..4c41ff4a9 --- /dev/null +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolutionInfoParser.kt @@ -0,0 +1,60 @@ +package org.jetbrains.kotlin.jupyter.libraries + +import org.jetbrains.kotlin.jupyter.ReplCompilerException +import org.jetbrains.kotlin.jupyter.Variable +import kotlin.script.experimental.api.ResultWithDiagnostics +import kotlin.script.experimental.api.asSuccess + +abstract class LibraryResolutionInfoParser(val name: String, private val parameters: List) { + fun getInfo(args: List): LibraryResolutionInfo { + val map = when(val mapResult = substituteArguments(parameters, args)) { + is ResultWithDiagnostics.Success -> mapResult.value + is ResultWithDiagnostics.Failure -> throw ReplCompilerException(mapResult) + } + + return getInfo(map) + } + + abstract fun getInfo(args: Map): LibraryResolutionInfo + + companion object { + fun make(name: String, parameters: List, getInfo: (Map) -> LibraryResolutionInfo): LibraryResolutionInfoParser { + return object : LibraryResolutionInfoParser(name, parameters) { + override fun getInfo(args: Map): LibraryResolutionInfo = getInfo(args) + } + } + + private fun substituteArguments(parameters: List, arguments: List): ResultWithDiagnostics> { + val result = mutableMapOf() + val possibleParamsNames = parameters.map { it.name }.toHashSet() + + var argIndex = 0 + + for (arg in arguments) { + val param = parameters.getOrNull(argIndex) ?: + return diagFailure("Too many arguments for library resolution info: ${arguments.size} got, ${parameters.size} allowed") + if (arg.name.isNotEmpty()) break + + result[param.name] = arg.value + argIndex++ + } + + for (i in argIndex until arguments.size) { + val arg = arguments[i] + if (arg.name.isEmpty()) return diagFailure("Positional arguments in library resolution info shouldn't appear after keyword ones") + if (arg.name !in possibleParamsNames) return diagFailure("There is no such argument: ${arg.name}") + + result[arg.name] = arg.value + } + + parameters.forEach { + if (!result.containsKey(it.name)) { + if (it is Parameter.Optional) result[it.name] = it.default + else return diagFailure("Parameter ${it.name} is required, but was not specified") + } + } + + return result.asSuccess() + } + } +} diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolver.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolver.kt new file mode 100644 index 000000000..0d2299b0b --- /dev/null +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryResolver.kt @@ -0,0 +1,80 @@ +package org.jetbrains.kotlin.jupyter.libraries + +import org.jetbrains.kotlin.jupyter.LibraryDescriptor +import org.jetbrains.kotlin.jupyter.LibraryDescriptorExt +import org.jetbrains.kotlin.jupyter.LocalCacheDir +import org.jetbrains.kotlin.jupyter.LocalSettingsPath +import org.jetbrains.kotlin.jupyter.log +import java.nio.file.Paths + +abstract class LibraryResolver(private val parent: LibraryResolver? = null) { + protected abstract fun tryResolve(reference: LibraryReference): LibraryDescriptor? + protected abstract fun save(reference: LibraryReference, descriptor: LibraryDescriptor) + protected open fun shouldResolve(reference: LibraryReference): Boolean = true + + open val cache: Map? = null + + fun resolve(reference: LibraryReference): LibraryDescriptor? { + val shouldBeResolved = shouldResolve(reference) + if (shouldBeResolved) { + val result = tryResolve(reference) + if (result != null) { + return result + } + } + + val parentResult = parent?.resolve(reference) ?: return null + if (shouldBeResolved) { + save(reference, parentResult) + } + + return parentResult + } +} + +class FallbackLibraryResolver : LibraryResolver() { + override fun tryResolve(reference: LibraryReference): LibraryDescriptor? { + return reference.resolve() + } + + override fun save(reference: LibraryReference, descriptor: LibraryDescriptor) { + // fallback resolver doesn't cache results + } +} + +class LocalLibraryResolver( + parent: LibraryResolver?, + mainLibrariesDir: String +): LibraryResolver(parent) { + private val pathsToCheck = listOf( + Paths.get(LocalSettingsPath, LocalCacheDir).toString(), + LocalSettingsPath, + mainLibrariesDir + ) + + override fun shouldResolve(reference: LibraryReference): Boolean { + return reference.shouldBeCachedLocally + } + + override fun tryResolve(reference: LibraryReference): LibraryDescriptor? { + val jsons = pathsToCheck.mapNotNull { dir -> + val file = reference.getFile(dir) + if (!file.exists()) null + else { file.readText() } + } + + if (jsons.size > 1) log.warn("More than one file for library $reference found in local cache directories") + + val json = jsons.firstOrNull() ?: return null + return parseLibraryDescriptor(json) + } + + override fun save(reference: LibraryReference, descriptor: LibraryDescriptor) { + val dir = pathsToCheck.first() + val file = reference.getFile(dir) + file.parentFile.mkdirs() + file.writeText(descriptor.originalJson.toJsonString(true)) + } + + private fun LibraryReference.getFile(dir: String) = Paths.get(dir, this.key + "." + LibraryDescriptorExt).toFile() +} diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/util.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/util.kt new file mode 100644 index 000000000..09f585171 --- /dev/null +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/util.kt @@ -0,0 +1,136 @@ +package org.jetbrains.kotlin.jupyter.libraries + +import com.beust.klaxon.JsonObject +import com.beust.klaxon.Parser +import org.jetbrains.kotlin.jupyter.GitHubApiPrefix +import org.jetbrains.kotlin.jupyter.LibrariesDir +import org.jetbrains.kotlin.jupyter.LibraryDescriptor +import org.jetbrains.kotlin.jupyter.ReplCompilerException +import org.jetbrains.kotlin.jupyter.TypeHandler +import org.jetbrains.kotlin.jupyter.Variable +import org.jetbrains.kotlin.jupyter.catchAll +import org.jetbrains.kotlin.jupyter.getHttp +import org.jetbrains.kotlin.jupyter.log +import org.json.JSONObject +import java.io.File +import kotlin.script.experimental.api.ResultWithDiagnostics +import kotlin.script.experimental.api.ScriptDiagnostic + +sealed class Parameter(val name: String, open val default: String?) { + class Required(name: String): Parameter(name, null) + class Optional(name: String, override val default: String): Parameter(name, default) +} + +class Brackets(val open: Char, val close: Char) { + companion object { + val ROUND = Brackets('(', ')') + val SQUARE = Brackets('[', ']') + } +} + +enum class DefaultInfoSwitch { + GIT_REFERENCE, DIRECTORY +} + +class LibraryFactoryDefaultInfoSwitcher(private val libraryFactory: LibraryFactory, initialSwitchVal: T, private val switcher: (T) -> LibraryResolutionInfo) { + private val defaultInfoCache = hashMapOf() + + var switch: T = initialSwitchVal + set(value) { + libraryFactory.defaultResolutionInfo = defaultInfoCache.getOrPut(value) { switcher(value) } + field = value + } + + companion object { + fun default(factory: LibraryFactory, defaultDir: File, defaultRef: String): LibraryFactoryDefaultInfoSwitcher { + val initialInfo = factory.defaultResolutionInfo + val dirInfo = if (initialInfo is LibraryResolutionInfo.ByDir) initialInfo else LibraryResolutionInfo.ByDir(defaultDir) + val refInfo = if (initialInfo is LibraryResolutionInfo.ByGitRef) initialInfo else LibraryResolutionInfo.getInfoByRef(defaultRef) + return LibraryFactoryDefaultInfoSwitcher(factory, DefaultInfoSwitch.DIRECTORY) { switch -> + when(switch) { + DefaultInfoSwitch.DIRECTORY -> dirInfo + DefaultInfoSwitch.GIT_REFERENCE -> refInfo + } + } + } + } +} + +fun diagFailure(message: String): ResultWithDiagnostics.Failure { + return ResultWithDiagnostics.Failure(ScriptDiagnostic(ScriptDiagnostic.unspecifiedError, message)) +} + +fun parseLibraryArgument(str: String): Variable { + val eq = str.indexOf('=') + return if (eq == -1) Variable("", str.trim()) + else Variable(str.substring(0, eq).trim(), str.substring(eq + 1).trim()) +} + +fun parseCall(str: String, brackets: Brackets): Pair> { + val openBracketIndex = str.indexOf(brackets.open) + if (openBracketIndex == -1) return str.trim() to emptyList() + val name = str.substring(0, openBracketIndex).trim() + val args = str.substring(openBracketIndex + 1, str.indexOf(brackets.close, openBracketIndex)) + .split(',') + .map(::parseLibraryArgument) + return name to args +} + +fun parseLibraryName(str: String): Pair> { + return parseCall(str, Brackets.ROUND) +} + +fun getLatestCommitToLibraries(ref: String, sinceTimestamp: String?): Pair? = + log.catchAll { + var url = "$GitHubApiPrefix/commits?path=$LibrariesDir&sha=$ref" + if (sinceTimestamp != null) + url += "&since=$sinceTimestamp" + log.info("Checking for new commits to library descriptors at $url") + val arr = getHttp(url).jsonArray + if (arr.length() == 0) { + if (sinceTimestamp != null) + getLatestCommitToLibraries(ref, null) + else { + log.info("Didn't find any commits to '$LibrariesDir' at $url") + null + } + } else { + val commit = arr[0] as JSONObject + val sha = commit["sha"] as String + val timestamp = ((commit["commit"] as JSONObject)["committer"] as JSONObject)["date"] as String + sha to timestamp + } + } + +fun parseLibraryDescriptor(json: String): LibraryDescriptor { + val jsonParser = Parser.default() + val res = jsonParser.parse(json.byteInputStream()) + if (res is JsonObject) return parseLibraryDescriptor(res) + + throw ReplCompilerException("Result of library descriptor parsing is of type ${res.javaClass.canonicalName} which is unexpected") +} + +fun parseLibraryDescriptor(json: JsonObject): LibraryDescriptor { + return LibraryDescriptor( + originalJson = json, + dependencies = json.array("dependencies")?.toList().orEmpty(), + variables = json.obj("properties")?.map { Variable(it.key, it.value.toString()) }.orEmpty(), + imports = json.array("imports")?.toList().orEmpty(), + repositories = json.array("repositories")?.toList().orEmpty(), + init = json.array("init")?.toList().orEmpty(), + shutdown = json.array("shutdown")?.toList().orEmpty(), + initCell = json.array("initCell")?.toList().orEmpty(), + renderers = json.obj("renderers")?.map { TypeHandler(it.key, it.value.toString()) }?.toList().orEmpty(), + link = json.string("link"), + minKernelVersion = json.string("minKernelVersion"), + converters = json.obj("typeConverters")?.map { TypeHandler(it.key, it.value.toString()) }.orEmpty(), + annotations = json.obj("annotationHandlers")?.map { TypeHandler(it.key, it.value.toString()) }.orEmpty() + ) +} + +fun parseLibraryDescriptors(libJsons: Map): Map { + return libJsons.mapValues { + log.info("Parsing '${it.key}' descriptor") + parseLibraryDescriptor(it.value) + } +} diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt index 9afb15d13..8ee333dc1 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt @@ -6,12 +6,16 @@ import com.github.ajalt.clikt.parameters.options.flag import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.long +import org.jetbrains.kotlin.jupyter.libraries.DefaultInfoSwitch +import org.jetbrains.kotlin.jupyter.libraries.LibrariesProcessor +import org.jetbrains.kotlin.jupyter.libraries.LibraryFactoryDefaultInfoSwitcher enum class ReplLineMagics(val desc: String, val argumentsUsage: String? = null, val visibleInHelp: Boolean = true) { use("include supported libraries", "klaxon(5.0.1), lets-plot"), trackClasspath("log current classpath changes"), trackExecution("log code that is going to be executed in repl", visibleInHelp = false), dumpClassesForSpark("stores compiled repl classes in special folder for Spark integration", visibleInHelp = false), + useLatestDescriptors("Download latest versions of library descriptors for the current branch", "-[on|off]"), output("setup output settings", "--max-cell-size=1000 --no-stdout --max-time=100 --max-buffer=400"); companion object { @@ -29,6 +33,8 @@ data class MagicProcessingResult(val code: String, val libraries: List): OutputConfig { val parser = object : CliktCommand() { @@ -54,7 +60,7 @@ class MagicsProcessor(val repl: ReplOptions, private val libraries: LibrariesPro } } - fun processMagics(code: String, ignoreMagicsErrors: Boolean = false): MagicProcessingResult { + fun processMagics(code: String, parseOnly: Boolean = false, tryIgnoreErrors: Boolean = false): MagicProcessingResult { val sb = StringBuilder() var nextSearchIndex = 0 @@ -81,8 +87,8 @@ class MagicsProcessor(val repl: ReplOptions, private val libraries: LibrariesPro val keyword = parts[0] val arg = if (parts.count() > 1) parts[1] else null - val magic = ReplLineMagics.valueOfOrNull(keyword) - if(magic == null && !ignoreMagicsErrors) { + val magic = if (parseOnly) null else ReplLineMagics.valueOfOrNull(keyword) + if(magic == null && !parseOnly && !tryIgnoreErrors) { throw ReplCompilerException("Unknown line magic keyword: '$keyword'") } @@ -104,8 +110,15 @@ class MagicsProcessor(val repl: ReplOptions, private val libraries: LibrariesPro try { if (arg == null) throw ReplCompilerException("Need some arguments for 'use' command") newLibraries.addAll(libraries.processNewLibraries(arg)) - } catch (e: ReplCompilerException) { - if (!ignoreMagicsErrors) throw e + } catch (e: Exception) { + if (!tryIgnoreErrors) throw e + } + } + ReplLineMagics.useLatestDescriptors -> { + libraryResolutionInfoSwitcher.switch = when(arg?.trim()) { + "-on" -> DefaultInfoSwitch.GIT_REFERENCE + "-off" -> DefaultInfoSwitch.DIRECTORY + else -> DefaultInfoSwitch.GIT_REFERENCE } } ReplLineMagics.output -> { @@ -119,4 +132,4 @@ class MagicsProcessor(val repl: ReplOptions, private val libraries: LibrariesPro } } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/message.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/message.kt index 278ff7cc0..e6807687a 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/message.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/message.kt @@ -50,4 +50,4 @@ fun makeHeader(msgType: String? = null, incomingMsg: Message? = null, sessionId: "version" to protocolVersion, "username" to ((incomingMsg?.header?.get("username") as? String) ?: "kernel"), "session" to ((incomingMsg?.header?.get("session") as? String) ?: sessionId), - "msg_type" to (msgType ?: "none")) \ No newline at end of file + "msg_type" to (msgType ?: "none")) diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt index 3376cba98..b99725fee 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt @@ -67,7 +67,21 @@ data class ErrorResponseWithMessage( override val hasStdErr: Boolean = stdErr != null && stdErr.isNotEmpty() } -fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJupyter?, executionCount: AtomicLong) { +fun JupyterConnection.Socket.controlMessagesHandler(msg: Message, repl: ReplForJupyter?) { + when(msg.header!!["msg_type"]) { + "interrupt_request" -> { + log.warn("Interruption is not yet supported!") + send(makeReplyMessage(msg, "interrupt_reply", content = msg.content)) + } + "shutdown_request" -> { + repl?.evalOnShutdown() + send(makeReplyMessage(msg, "shutdown_reply", content = msg.content)) + exitProcess(0) + } + } +} + +fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJupyter, executionCount: AtomicLong) { when (msg.header!!["msg_type"]) { "kernel_info_request" -> sendWrapped(msg, makeReplyMessage(msg, "kernel_info_reply", @@ -85,9 +99,9 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup ), // Jupyter lab Console support - "banner" to "Kotlin kernel v. ${runtimeProperties.version.toMaybeUnspecifiedString()}, Kotlin v. ${KotlinCompilerVersion.VERSION}", + "banner" to "Kotlin kernel v. ${repl.runtimeProperties.version.toMaybeUnspecifiedString()}, Kotlin v. ${KotlinCompilerVersion.VERSION}", "implementation" to "Kotlin", - "implementation_version" to runtimeProperties.version.toMaybeUnspecifiedString(), + "implementation_version" to repl.runtimeProperties.version.toMaybeUnspecifiedString(), "status" to "ok" ))) "history_request" -> @@ -95,15 +109,6 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup content = jsonObject( "history" to listOf() // not implemented ))) - "interrupt_request" -> { - log.warn("Interruption is not yet supported!") - send(makeReplyMessage(msg, "interrupt_reply", content = msg.content)) - } - "shutdown_request" -> { - repl?.evalOnShutdown() - send(makeReplyMessage(msg, "shutdown_reply", content = msg.content)) - exitProcess(0) - } // TODO: This request is deprecated since messaging protocol v.5.1, // remove it in future versions of kernel @@ -134,7 +139,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup val res: Response = if (isCommand(code.toString())) { runCommand(code.toString(), repl) } else { - connection.evalWithIO(repl!!.outputConfig, msg) { + connection.evalWithIO(repl.outputConfig, msg) { repl.eval(code.toString(), ::displayHandler, count.toInt()) } } @@ -204,10 +209,6 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup "complete_request" -> { val code = msg.content["code"].toString() val cursor = msg.content["cursor_pos"] as Int - if (repl == null) { - System.err.println("Repl is not yet initialized on complete request") - return - } GlobalScope.launch { repl.complete(code, cursor) { result -> sendWrapped(msg, makeReplyMessage(msg, "complete_reply", content = result.toJson())) @@ -216,10 +217,6 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup } "list_errors_request" -> { val code = msg.content["code"].toString() - if (repl == null) { - System.err.println("Repl is not yet initialized on listErrors request") - return - } GlobalScope.launch { repl.listErrors(code) { result -> sendWrapped(msg, makeReplyMessage(msg, "list_errors_reply", content = result.toJson())) @@ -230,9 +227,8 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup val code = msg.content["code"].toString() val resStr = if (isCommand(code)) "complete" else { val result = try { - val check = repl?.checkComplete(code) + val check = repl.checkComplete(code) when { - check == null -> "error: no repl" check.isComplete -> "complete" else -> "incomplete" } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt index 363e29cd5..208ef8b0b 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt @@ -1,27 +1,63 @@ package org.jetbrains.kotlin.jupyter -import jupyter.kotlin.* +import jupyter.kotlin.DependsOn import jupyter.kotlin.KotlinContext +import jupyter.kotlin.KotlinKernelHost +import jupyter.kotlin.KotlinKernelVersion +import jupyter.kotlin.ReplOutputs +import jupyter.kotlin.Repository +import jupyter.kotlin.ScriptTemplateWithDisplayHelpers import kotlinx.coroutines.runBlocking import org.jetbrains.kotlin.config.KotlinCompilerVersion -import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResult -import org.jetbrains.kotlin.jupyter.repl.completion.KotlinCompleter -import org.jetbrains.kotlin.jupyter.repl.completion.ListErrorsResult -import org.jetbrains.kotlin.jupyter.repl.completion.SourceCodeImpl -import org.jetbrains.kotlin.jupyter.repl.reflect.ContextUpdater -import org.jetbrains.kotlin.jupyter.repl.spark.ClassWriter +import org.jetbrains.kotlin.jupyter.libraries.LibrariesProcessor +import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory +import org.jetbrains.kotlin.jupyter.repl.ClassWriter +import org.jetbrains.kotlin.jupyter.repl.CompletionResult +import org.jetbrains.kotlin.jupyter.repl.ContextUpdater +import org.jetbrains.kotlin.jupyter.repl.KotlinCompleter +import org.jetbrains.kotlin.jupyter.repl.ListErrorsResult +import org.jetbrains.kotlin.jupyter.repl.SourceCodeImpl import org.jetbrains.kotlin.scripting.ide_services.compiler.KJvmReplCompilerWithIdeServices import org.jetbrains.kotlin.scripting.resolve.skipExtensionsResolutionForImplicitsExceptInnermost import java.io.File import java.net.URLClassLoader import java.util.* import kotlin.script.dependencies.ScriptContents -import kotlin.script.experimental.api.* +import kotlin.script.experimental.api.KotlinType +import kotlin.script.experimental.api.ReplAnalyzerResult +import kotlin.script.experimental.api.ResultValue +import kotlin.script.experimental.api.ResultWithDiagnostics +import kotlin.script.experimental.api.ScriptCollectedData +import kotlin.script.experimental.api.ScriptCompilationConfiguration +import kotlin.script.experimental.api.ScriptConfigurationRefinementContext +import kotlin.script.experimental.api.ScriptDiagnostic +import kotlin.script.experimental.api.ScriptEvaluationConfiguration +import kotlin.script.experimental.api.analysisDiagnostics +import kotlin.script.experimental.api.asDiagnostics +import kotlin.script.experimental.api.asSuccess +import kotlin.script.experimental.api.baseClass +import kotlin.script.experimental.api.compilerOptions +import kotlin.script.experimental.api.constructorArgs +import kotlin.script.experimental.api.defaultImports +import kotlin.script.experimental.api.dependencies +import kotlin.script.experimental.api.fileExtension +import kotlin.script.experimental.api.foundAnnotations +import kotlin.script.experimental.api.hostConfiguration +import kotlin.script.experimental.api.implicitReceivers +import kotlin.script.experimental.api.onSuccess +import kotlin.script.experimental.api.refineConfiguration +import kotlin.script.experimental.api.valueOrThrow import kotlin.script.experimental.host.withDefaultsFrom -import kotlin.script.experimental.jvm.* +import kotlin.script.experimental.jvm.BasicJvmReplEvaluator +import kotlin.script.experimental.jvm.JvmDependency +import kotlin.script.experimental.jvm.baseClassLoader +import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration +import kotlin.script.experimental.jvm.jvm +import kotlin.script.experimental.jvm.updateClasspath import kotlin.script.experimental.jvm.util.isError import kotlin.script.experimental.jvm.util.isIncomplete import kotlin.script.experimental.jvm.util.toSourceCodePosition +import kotlin.script.experimental.jvm.withUpdatedClasspath data class EvalResult(val resultValue: Any?) @@ -47,13 +83,21 @@ enum class ExecutedCodeLogging { Generated } +interface ReplRuntimeProperties { + val version: KotlinKernelVersion? + val librariesFormatVersion: Int + val currentBranch: String + val currentSha: String + val jvmTargetForSnippets: String +} + interface ReplOptions { - var trackClasspath: Boolean + val currentBranch: String + val librariesDir: File + var trackClasspath: Boolean var executedCodeLogging: ExecutedCodeLogging - var writeCompiledClasses: Boolean - var outputConfig: OutputConfig } @@ -71,15 +115,34 @@ interface ReplForJupyter { suspend fun listErrors(code: String, callback: (ListErrorsResult) -> Unit) + val homeDir: File? + val currentClasspath: Collection val resolverConfig: ResolverConfig? + val runtimeProperties: ReplRuntimeProperties + + val libraryFactory: LibraryFactory + var outputConfig: OutputConfig } -class ReplForJupyterImpl(private val scriptClasspath: List = emptyList(), - override val resolverConfig: ResolverConfig? = null, vararg scriptReceivers: Any) : ReplForJupyter, ReplOptions, KotlinKernelHost { +class ReplForJupyterImpl( + override val libraryFactory: LibraryFactory, + private val scriptClasspath: List = emptyList(), + override val homeDir: File? = null, + override val resolverConfig: ResolverConfig? = null, + override val runtimeProperties: ReplRuntimeProperties = defaultRuntimeProperties, + private val scriptReceivers: List = emptyList(), +) : ReplForJupyter, ReplOptions, KotlinKernelHost { + + constructor(config: KernelConfig, runtimeProperties: ReplRuntimeProperties, scriptReceivers: List = emptyList()): + this(config.libraryFactory, config.scriptClasspath, config.homeDir, config.resolverConfig, runtimeProperties, scriptReceivers) + + override val currentBranch: String + get() = runtimeProperties.currentBranch + override val librariesDir: File = homeDir?.resolve(LibrariesDir) ?: File(LibrariesDir) private var outputConfigImpl = OutputConfig() @@ -177,9 +240,7 @@ class ReplForJupyterImpl(private val scriptClasspath: List = emptyList(), private val ctx = KotlinContext() - private val receivers: List = scriptReceivers.asList() - - private val magics = MagicsProcessor(this, LibrariesProcessor(resolverConfig?.libraries)) + private val magics = MagicsProcessor(this, LibrariesProcessor(resolverConfig?.libraries, runtimeProperties, libraryFactory)) private fun configureMavenDepsOnAnnotations(context: ScriptConfigurationRefinementContext): ResultWithDiagnostics { val annotations = context.collectedData?.get(ScriptCollectedData.foundAnnotations)?.takeIf { it.isNotEmpty() } @@ -221,7 +282,7 @@ class ReplForJupyterImpl(private val scriptClasspath: List = emptyList(), onAnnotations(DependsOn::class, Repository::class, handler = { configureMavenDepsOnAnnotations(it) }) } - val receiversTypes = receivers.map { KotlinType(it.javaClass.canonicalName) } + val receiversTypes = scriptReceivers.map { KotlinType(it.javaClass.canonicalName) } implicitReceivers(receiversTypes) skipExtensionsResolutionForImplicitsExceptInnermost(receiversTypes) @@ -254,7 +315,7 @@ class ReplForJupyterImpl(private val scriptClasspath: List = emptyList(), } private val evaluatorConfiguration = ScriptEvaluationConfiguration { - implicitReceivers.invoke(v = receivers) + implicitReceivers.invoke(v = scriptReceivers) jvm { val filteringClassLoader = FilteringClassLoader(ClassLoader.getSystemClassLoader()) { it.startsWith("jupyter.kotlin.") || it.startsWith("kotlin.") || (it.startsWith("org.jetbrains.kotlin.") && !it.startsWith("org.jetbrains.kotlin.jupyter.")) @@ -583,4 +644,3 @@ class ReplForJupyterImpl(private val scriptClasspath: List = emptyList(), scheduledExecutions.add(code) } } - diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/spark/ClassWriter.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/ClassWriter.kt similarity index 97% rename from src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/spark/ClassWriter.kt rename to src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/ClassWriter.kt index 1b7405a22..a0dc555c8 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/spark/ClassWriter.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/ClassWriter.kt @@ -1,4 +1,4 @@ -package org.jetbrains.kotlin.jupyter.repl.spark +package org.jetbrains.kotlin.jupyter.repl import org.slf4j.LoggerFactory import java.io.BufferedOutputStream diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/reflect/ContextUpdater.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/ContextUpdater.kt similarity index 98% rename from src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/reflect/ContextUpdater.kt rename to src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/ContextUpdater.kt index 7ce8ebf96..5a5d6f231 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/reflect/ContextUpdater.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/ContextUpdater.kt @@ -1,4 +1,4 @@ -package org.jetbrains.kotlin.jupyter.repl.reflect +package org.jetbrains.kotlin.jupyter.repl import jupyter.kotlin.KotlinContext import jupyter.kotlin.KotlinFunctionInfo diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/completion/KotlinCompleter.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/KotlinCompleter.kt similarity index 95% rename from src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/completion/KotlinCompleter.kt rename to src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/KotlinCompleter.kt index 4fe4a8e83..283a72df6 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/completion/KotlinCompleter.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/KotlinCompleter.kt @@ -1,4 +1,4 @@ -package org.jetbrains.kotlin.jupyter.repl.completion +package org.jetbrains.kotlin.jupyter.repl import com.beust.klaxon.JsonObject import kotlinx.coroutines.runBlocking @@ -8,8 +8,6 @@ import org.jetbrains.kotlin.jupyter.toSourceCodePositionWithNewAbsolute import java.io.PrintWriter import java.io.StringWriter import kotlin.script.experimental.api.* -import kotlin.script.experimental.jvm.util.calcAbsolute -import kotlin.script.experimental.jvm.util.toSourceCodePosition enum class CompletionStatus(private val value: String) { OK("ok"), @@ -76,7 +74,7 @@ abstract class CompletionResult( class Empty( text: String, cursor: Int - ): CompletionResult.Success(emptyList(), CompletionTokenBounds(cursor, cursor), emptyList(), text, cursor) + ): Success(emptyList(), CompletionTokenBounds(cursor, cursor), emptyList(), text, cursor) class Error( private val errorName: String, diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/typeProviders.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/typeProviders.kt index c0603ddb3..0ad61c391 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/typeProviders.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/typeProviders.kt @@ -1,7 +1,7 @@ package org.jetbrains.kotlin.jupyter import jupyter.kotlin.KotlinFunctionInfo -import org.jetbrains.kotlin.jupyter.repl.reflect.ContextUpdater +import org.jetbrains.kotlin.jupyter.repl.ContextUpdater import kotlin.reflect.KMutableProperty import kotlin.reflect.KProperty import kotlin.reflect.full.withNullability @@ -144,4 +144,4 @@ class TypeProvidersProcessorImpl(private val contextUpdater: ContextUpdater) : T } return emptyList() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt index cd1df0535..f1956b0b1 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt @@ -1,25 +1,14 @@ package org.jetbrains.kotlin.jupyter -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.async -import kotlinx.coroutines.runBlocking import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocationWithRange -import org.jetbrains.kotlin.jupyter.repl.completion.SourceCodeImpl +import org.jetbrains.kotlin.jupyter.repl.SourceCodeImpl import org.slf4j.Logger -import java.io.File import kotlin.script.experimental.api.ResultWithDiagnostics import kotlin.script.experimental.api.ScriptDiagnostic import kotlin.script.experimental.api.SourceCode import kotlin.script.experimental.jvm.util.determineSep import kotlin.script.experimental.jvm.util.toSourceCodePosition -fun catchAll(body: () -> T): T? = try { - body() -} catch (e: Exception) { - null -} - fun Logger.catchAll(msg: String = "", body: () -> T): T? = try { body() } catch (e: Exception) { @@ -27,24 +16,11 @@ fun Logger.catchAll(msg: String = "", body: () -> T): T? = try { null } -fun T.validOrNull(predicate: (T) -> Boolean): T? = if (predicate(this)) this else null - -fun T.asAsync(): Deferred = GlobalScope.async { this@asAsync } - -fun File.existsOrNull() = if (exists()) this else null - -fun Deferred.awaitBlocking(): T = if (isCompleted) getCompleted() else runBlocking { await() } - fun String.parseIniConfig() = lineSequence().map { it.split('=') }.filter { it.count() == 2 }.map { it[0] to it[1] }.toMap() fun List.joinToLines() = joinToString("\n") -fun File.tryReadIniConfig() = - existsOrNull()?.let { - catchAll { it.readText().parseIniConfig() } - } - fun generateDiagnostic(fromLine: Int, fromCol: Int, toLine: Int, toCol: Int, message: String, severity: String) = ScriptDiagnostic( diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/configTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/configTests.kt index 57b092e19..9833029fb 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/configTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/configTests.kt @@ -2,7 +2,11 @@ package org.jetbrains.kotlin.jupyter.test import jupyter.kotlin.JavaRuntime import jupyter.kotlin.KotlinKernelVersion -import org.jetbrains.kotlin.jupyter.* +import org.jetbrains.kotlin.jupyter.LibrariesDir +import org.jetbrains.kotlin.jupyter.LibraryPropertiesFile +import org.jetbrains.kotlin.jupyter.defaultRuntimeProperties +import org.jetbrains.kotlin.jupyter.log +import org.jetbrains.kotlin.jupyter.parseIniConfig import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue @@ -13,7 +17,7 @@ import kotlin.test.assertNotNull class ConfigTest { @Test fun testBranch() { - val branch = runtimeProperties.currentBranch + val branch = defaultRuntimeProperties.currentBranch log.debug("Runtime git branch is: $branch") if (!branch.matches(Regex("pull/[1-9][0-9]*"))) @@ -21,13 +25,13 @@ class ConfigTest { assertTrue(branch.isNotBlank(), "Branch name shouldn't be blank") - val commit = runtimeProperties.currentSha + val commit = defaultRuntimeProperties.currentSha assertEquals(40, commit.length) } @Test fun testLibrariesProperties() { - val format = runtimeProperties.librariesFormatVersion + val format = defaultRuntimeProperties.librariesFormatVersion log.debug("Runtime libs format is: $format") assertTrue(format in 2..1000) @@ -37,7 +41,7 @@ class ConfigTest { @Test fun testVersion() { - val version = runtimeProperties.version + val version = defaultRuntimeProperties.version log.debug("Runtime version is: $version") assertNotNull(version) diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/ikotlinTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/ikotlinTests.kt index b6998aed9..cec7dd3f7 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/ikotlinTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/ikotlinTests.kt @@ -46,9 +46,9 @@ class KernelServerTest : KernelServerTestsBase() { with (ClientSocket(context, JupyterSockets.control)) { try { connect() - sendMessage(Message(id = messageId, header = makeHeader("kernel_info_request")), hmac) + sendMessage(Message(id = messageId, header = makeHeader("interrupt_request")), hmac) val msg = receiveMessage(recv(), hmac) - assertEquals("kernel_info_reply", msg!!.header!!["msg_type"]) + assertEquals("interrupt_reply", msg!!.header!!["msg_type"]) } finally { close() context.term() diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/kernelServerTestsBase.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/kernelServerTestsBase.kt index b01eed321..1d9a610fe 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/kernelServerTestsBase.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/kernelServerTestsBase.kt @@ -5,8 +5,11 @@ import org.jetbrains.kotlin.jupyter.HMAC import org.jetbrains.kotlin.jupyter.JupyterSockets import org.jetbrains.kotlin.jupyter.KernelConfig import org.jetbrains.kotlin.jupyter.Message +import org.jetbrains.kotlin.jupyter.defaultRuntimeProperties import org.jetbrains.kotlin.jupyter.iKotlinClass import org.jetbrains.kotlin.jupyter.kernelServer +import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory +import org.jetbrains.kotlin.jupyter.libraries.LibraryResolutionInfo import org.jetbrains.kotlin.jupyter.makeHeader import org.jetbrains.kotlin.jupyter.receiveMessage import org.jetbrains.kotlin.jupyter.sendMessage @@ -33,6 +36,8 @@ open class KernelServerTestsBase { signatureKey = "", scriptClasspath = classpath, resolverConfig = null, + homeDir = File(""), + libraryFactory = LibraryFactory(LibraryResolutionInfo.ByNothing()) ) private val sessionId = UUID.randomUUID().toString() @@ -72,7 +77,7 @@ open class KernelServerTestsBase { .redirectError(fileErr) .start() } else { - serverThread = thread { kernelServer(config) } + serverThread = thread { kernelServer(config, defaultRuntimeProperties) } } } diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseMagicsTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseMagicsTests.kt index 255af4830..2acda029b 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseMagicsTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseMagicsTests.kt @@ -1,32 +1,37 @@ package org.jetbrains.kotlin.jupyter.test import org.jetbrains.kotlin.jupyter.ExecutedCodeLogging -import org.jetbrains.kotlin.jupyter.LibrariesProcessor +import org.jetbrains.kotlin.jupyter.LibrariesDir +import org.jetbrains.kotlin.jupyter.libraries.LibrariesProcessor import org.jetbrains.kotlin.jupyter.LibraryDefinition import org.jetbrains.kotlin.jupyter.MagicsProcessor import org.jetbrains.kotlin.jupyter.OutputConfig import org.jetbrains.kotlin.jupyter.ReplOptions -import org.jetbrains.kotlin.jupyter.parseLibraryName -import org.jetbrains.kotlin.jupyter.repl.completion.SourceCodeImpl +import org.jetbrains.kotlin.jupyter.defaultRuntimeProperties +import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory +import org.jetbrains.kotlin.jupyter.libraries.LibraryResolutionInfo +import org.jetbrains.kotlin.jupyter.repl.SourceCodeImpl import org.jetbrains.kotlin.jupyter.toSourceCodePositionWithNewAbsolute import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import java.io.File +import kotlin.test.assertTrue class ParseArgumentsTests { + private val libraryFactory = LibraryFactory(LibraryResolutionInfo.ByNothing()) @Test fun test1() { - val (name, args) = parseLibraryName(" lib ") - assertEquals("lib", name) + val (ref, args) = libraryFactory.parseReferenceWithArgs(" lib ") + assertEquals("lib", ref.name) assertEquals(0, args.count()) } @Test fun test2() { - val (name, args) = parseLibraryName("lib(arg1)") - assertEquals("lib", name) + val (ref, args) = libraryFactory.parseReferenceWithArgs("lib(arg1)") + assertEquals("lib", ref.name) assertEquals(1, args.count()) assertEquals("arg1", args[0].value) assertEquals("", args[0].name) @@ -34,19 +39,59 @@ class ParseArgumentsTests { @Test fun test3() { - val (name, args) = parseLibraryName("lib (arg1 = 1.2, arg2 = val2)") - assertEquals("lib", name) + val (ref, args) = libraryFactory.parseReferenceWithArgs("lib (arg1 = 1.2, arg2 = val2)") + assertEquals("lib", ref.name) assertEquals(2, args.count()) assertEquals("arg1", args[0].name) assertEquals("1.2", args[0].value) assertEquals("arg2", args[1].name) assertEquals("val2", args[1].value) } + + @Test + fun testInfo1() { + val requestUrl = "https://raw.githubusercontent.com/Kotlin/kotlin-jupyter/master/libraries/default.json" + val (ref, args) = libraryFactory.parseReferenceWithArgs("lib_name@url[$requestUrl]") + assertEquals("lib_name", ref.name) + + val info = ref.info + assertTrue(info is LibraryResolutionInfo.ByURL) + assertEquals(requestUrl, info.url.toString()) + assertEquals(0, args.size) + } + + @Test + fun testInfo2() { + val file = File("libraries/default.json").toString() + val (ref, args) = libraryFactory.parseReferenceWithArgs("@file[$file](param=val)") + assertEquals("", ref.name) + + val info = ref.info + assertTrue(info is LibraryResolutionInfo.ByFile) + assertEquals(file, info.file.toString()) + assertEquals(1, args.size) + assertEquals("param", args[0].name) + assertEquals("val", args[0].value) + } + + @Test + fun testInfo3() { + val (ref, args) = libraryFactory.parseReferenceWithArgs("krangl@0.8.2.5") + assertEquals("krangl", ref.name) + + val info = ref.info + assertTrue(info is LibraryResolutionInfo.ByGitRef) + assertEquals(40, info.sha.length) + assertEquals(0, args.size) + } } class ParseMagicsTests { private class TestReplOptions : ReplOptions { + override val currentBranch: String + get() = standardResolverBranch + override val librariesDir = File(LibrariesDir) override var trackClasspath = false override var executedCodeLogging = ExecutedCodeLogging.Off override var writeCompiledClasses = false @@ -56,8 +101,9 @@ class ParseMagicsTests { private val options = TestReplOptions() private fun test(code: String, expectedProcessedCode: String, librariesChecker: (List) -> Unit = {}) { - val processor = MagicsProcessor(options, LibrariesProcessor(testResolverConfig.libraries)) - with(processor.processMagics(code, true)) { + val libraryFactory = LibraryFactory(LibraryResolutionInfo.ByNothing()) + val processor = MagicsProcessor(options, LibrariesProcessor(libraryFactory.testResolverConfig.libraries, defaultRuntimeProperties, libraryFactory)) + with(processor.processMagics(code, tryIgnoreErrors = true)) { assertEquals(expectedProcessedCode, this.code) librariesChecker(libraries) } diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt index 669cda7f2..753eee13b 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt @@ -5,17 +5,23 @@ import jupyter.kotlin.KotlinKernelVersion.Companion.toMaybeUnspecifiedString import jupyter.kotlin.MimeTypedResult import jupyter.kotlin.receivers.ConstReceiver import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.jupyter.GitHubRepoName +import org.jetbrains.kotlin.jupyter.GitHubRepoOwner +import org.jetbrains.kotlin.jupyter.LibrariesDir import org.jetbrains.kotlin.jupyter.OutputConfig import org.jetbrains.kotlin.jupyter.ReplCompilerException import org.jetbrains.kotlin.jupyter.ReplEvalRuntimeException import org.jetbrains.kotlin.jupyter.ReplForJupyterImpl import org.jetbrains.kotlin.jupyter.ResolverConfig import org.jetbrains.kotlin.jupyter.defaultRepositories +import org.jetbrains.kotlin.jupyter.defaultRuntimeProperties import org.jetbrains.kotlin.jupyter.generateDiagnostic import org.jetbrains.kotlin.jupyter.generateDiagnosticFromAbsolute -import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResult -import org.jetbrains.kotlin.jupyter.repl.completion.ListErrorsResult -import org.jetbrains.kotlin.jupyter.runtimeProperties +import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory +import org.jetbrains.kotlin.jupyter.libraries.LibraryResolutionInfo +import org.jetbrains.kotlin.jupyter.libraries.LibraryResolver +import org.jetbrains.kotlin.jupyter.repl.CompletionResult +import org.jetbrains.kotlin.jupyter.repl.ListErrorsResult import org.jetbrains.kotlin.jupyter.withPath import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -34,10 +40,18 @@ abstract class AbstractReplTest { protected fun String.convertCRLFtoLF(): String { return replace("\r\n", "\n") } + + companion object { + @JvmStatic + protected val libraryFactory = LibraryFactory(LibraryResolutionInfo.ByNothing()) + + @JvmStatic + protected val homeDir = File("") + } } class ReplTest : AbstractReplTest() { - private val repl = ReplForJupyterImpl(classpath) + private val repl = ReplForJupyterImpl(libraryFactory, classpath) @Test fun testRepl() { @@ -94,7 +108,7 @@ class ReplTest : AbstractReplTest() { fun testReplWithReceiver() { val value = 5 val cp = classpath + File(ConstReceiver::class.java.protectionDomain.codeSource.location.toURI().path) - val repl = ReplForJupyterImpl(cp, null, ConstReceiver(value)) + val repl = ReplForJupyterImpl(libraryFactory, cp, null, scriptReceivers = listOf(ConstReceiver(value))) val res = repl.eval("value") assertEquals(value, res.resultValue) } @@ -332,6 +346,24 @@ class ReplTest : AbstractReplTest() { assertEquals(OutputConfig(), repl.outputConfig) } + @Test + fun testJavaRuntimeUtils() { + val result = repl.eval("JavaRuntimeUtils.version") + val resultVersion = result.resultValue + val expectedVersion = JavaRuntime.version + assertEquals(expectedVersion, resultVersion) + } + + @Test + fun testKotlinMath() { + val result = repl.eval("2.0.pow(2.0)").resultValue + assertEquals(4.0, result) + } +} + +class CustomLibraryResolverTests : AbstractReplTest() { + private fun makeRepl(libs: LibraryResolver) = ReplForJupyterImpl(libraryFactory, classpath, homeDir, ResolverConfig(defaultRepositories, libs)) + @Test fun testUseMagic() { val lib1 = "mylib" to """ @@ -390,9 +422,9 @@ class ReplTest : AbstractReplTest() { } """.trimIndent() - val libs = listOf(lib1, lib2, lib3).toLibrariesAsync() + val libs = listOf(lib1, lib2, lib3).toLibraries(libraryFactory) - val replWithResolver = ReplForJupyterImpl(classpath, ResolverConfig(defaultRepositories, libs)) + val replWithResolver = makeRepl(libs) val res = replWithResolver.preprocessCode("%use mylib(1.0), another") assertEquals("", res.code) val inits = arrayOf( @@ -440,8 +472,8 @@ class ReplTest : AbstractReplTest() { ] }""".trimIndent() - val libs = listOf(lib1, lib2).toLibrariesAsync() - val replWithResolver = ReplForJupyterImpl(classpath, ResolverConfig(defaultRepositories, libs)) + val libs = listOf(lib1, lib2).toLibraries(libraryFactory) + val replWithResolver = makeRepl(libs) replWithResolver.eval("%use mylib, mylib2") val results = replWithResolver.evalOnShutdown() @@ -455,40 +487,32 @@ class ReplTest : AbstractReplTest() { @Test fun testLibraryKernelVersionRequirements() { val minRequiredVersion = "999.42.0.1" - val kernelVersion = runtimeProperties.version.toMaybeUnspecifiedString() + val kernelVersion = defaultRuntimeProperties.version.toMaybeUnspecifiedString() val lib1 = "mylib" to """ { "minKernelVersion": "$minRequiredVersion" }""".trimIndent() - val libs = listOf(lib1).toLibrariesAsync() - val replWithResolver = ReplForJupyterImpl(classpath, ResolverConfig(defaultRepositories, libs)) + val libs = listOf(lib1).toLibraries(libraryFactory) + val replWithResolver = makeRepl(libs) val exception = assertThrows { replWithResolver.eval("%use mylib") } val message = exception.message!! assertTrue(message.contains(minRequiredVersion)) assertTrue(message.contains(kernelVersion)) } - - @Test - fun testJavaRuntimeUtils() { - val result = repl.eval("JavaRuntimeUtils.version") - val resultVersion = result.resultValue - val expectedVersion = JavaRuntime.version - assertEquals(expectedVersion, resultVersion) - } - - @Test - fun testKotlinMath() { - val result = repl.eval("2.0.pow(2.0)").resultValue - assertEquals(4.0, result) - } } @Execution(ExecutionMode.SAME_THREAD) class ReplWithResolverTest : AbstractReplTest() { - private val repl = ReplForJupyterImpl(classpath, testResolverConfig) + private val repl = ReplForJupyterImpl(libraryFactory, classpath, homeDir, resolverConfig) + + private fun getReplWithStandardResolver(): ReplForJupyterImpl { + val standardLibraryFactory = LibraryFactory.withDefaultDirectoryResolution(homeDir.resolve(LibrariesDir)) + val config = ResolverConfig(defaultRepositories, standardLibraryFactory.getStandardResolver(".")) + return ReplForJupyterImpl(standardLibraryFactory, classpath, homeDir, config, standardResolverRuntimeProperties) + } @Test fun testLetsPlot() { @@ -530,6 +554,58 @@ class ReplWithResolverTest : AbstractReplTest() { assertNotNull(res.resultValue) } + @Test + fun testStandardLibraryResolver() { + val repl = getReplWithStandardResolver() + + val res = repl.eval(""" + %use krangl(0.13) + val df = DataFrame.readCSV("src/test/testData/resolve-with-runtime.csv") + df.head().rows.first().let { it["name"].toString() + " " + it["surname"].toString() } + """.trimIndent()) + assertEquals("John Smith", res.resultValue) + } + + @Test + fun testDefaultInfoSwitcher() { + val repl = getReplWithStandardResolver() + + val initialDefaultResolutionInfo = repl.libraryFactory.defaultResolutionInfo + assertTrue(initialDefaultResolutionInfo is LibraryResolutionInfo.ByDir) + + repl.eval("%useLatestDescriptors") + assertTrue(repl.libraryFactory.defaultResolutionInfo is LibraryResolutionInfo.ByGitRef) + + repl.eval("%useLatestDescriptors -off") + assertTrue(repl.libraryFactory.defaultResolutionInfo === initialDefaultResolutionInfo) + } + + @Test + fun testUseFileUrlRef() { + val repl = getReplWithStandardResolver() + + val commit = "1f56d74a88f6fb78306d685d0b3aaf07113a8abf" + val libraryPath = "src/test/testData/test-init.json" + + val res1 = repl.eval(""" + %use @file[$libraryPath](name=x, value=42) + x + """.trimIndent()) + assertEquals(42, res1.resultValue) + + val res2 = repl.eval(""" + %use @url[https://raw.githubusercontent.com/$GitHubRepoOwner/$GitHubRepoName/$commit/$libraryPath](name=y, value=43) + y + """.trimIndent()) + assertEquals(43, res2.resultValue) + + val displays = mutableListOf() + val res3 = repl.eval("%use lets-plot@$commit", { displays.add(it) }) + assertEquals(1, displays.count()) + assertNull(res3.resultValue) + displays.clear() + } + @Test fun testRuntimeDepsResolution() { val res = repl.eval(""" @@ -564,4 +640,8 @@ class ReplWithResolverTest : AbstractReplTest() { """.trimIndent()) assertEquals(23, res.resultValue) } + + companion object { + val resolverConfig = libraryFactory.testResolverConfig + } } diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/testUtil.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/testUtil.kt index b3273226e..357d060bd 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/testUtil.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/testUtil.kt @@ -3,15 +3,28 @@ package org.jetbrains.kotlin.jupyter.test import com.beust.klaxon.JsonObject import com.beust.klaxon.Parser import jupyter.kotlin.DependsOn -import kotlinx.coroutines.Deferred +import org.jetbrains.kotlin.jupyter.LibrariesDir import org.jetbrains.kotlin.jupyter.LibraryDescriptor +import org.jetbrains.kotlin.jupyter.LibraryDescriptorExt +import org.jetbrains.kotlin.jupyter.ReplRuntimeProperties import org.jetbrains.kotlin.jupyter.ResolverConfig -import org.jetbrains.kotlin.jupyter.asAsync import org.jetbrains.kotlin.jupyter.defaultRepositories -import org.jetbrains.kotlin.jupyter.parserLibraryDescriptors -import org.jetbrains.kotlin.jupyter.readLibraries +import org.jetbrains.kotlin.jupyter.defaultRuntimeProperties +import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory +import org.jetbrains.kotlin.jupyter.libraries.LibraryReference +import org.jetbrains.kotlin.jupyter.libraries.LibraryResolver +import org.jetbrains.kotlin.jupyter.libraries.parseLibraryDescriptors +import org.jetbrains.kotlin.jupyter.log +import java.io.File import kotlin.script.experimental.jvm.util.scriptCompilationClasspathFromContext +const val standardResolverBranch = "master" + +val standardResolverRuntimeProperties = object : ReplRuntimeProperties by defaultRuntimeProperties { + override val currentBranch: String + get() = standardResolverBranch +} + val classpath = scriptCompilationClasspathFromContext( "jupyter-lib", "kotlin-stdlib", @@ -20,11 +33,52 @@ val classpath = scriptCompilationClasspathFromContext( classLoader = DependsOn::class.java.classLoader ) -val testResolverConfig = ResolverConfig(defaultRepositories, - parserLibraryDescriptors(readLibraries().toMap()).asAsync()) +val LibraryFactory.testResolverConfig: ResolverConfig + get() = ResolverConfig( + defaultRepositories, + getResolverFromNamesMap(parseLibraryDescriptors(readLibraries())) + ) -fun Collection>.toLibrariesAsync(): Deferred> { +fun Collection>.toLibraries(libraryFactory: LibraryFactory): LibraryResolver { val parser = Parser.default() val libJsons = map { it.first to parser.parse(StringBuilder(it.second)) as JsonObject }.toMap() - return parserLibraryDescriptors(libJsons).asAsync() + return libraryFactory.getResolverFromNamesMap(parseLibraryDescriptors(libJsons)) +} + +fun LibraryFactory.getResolverFromNamesMap(map: Map): LibraryResolver { + return InMemoryLibraryResolver(null, map.mapKeys { entry -> LibraryReference(defaultResolutionInfo, entry.key) }) +} + +fun readLibraries(basePath: String? = null): Map { + val parser = Parser.default() + return File(basePath, LibrariesDir) + .listFiles()?.filter { it.extension == LibraryDescriptorExt} + ?.map { + log.info("Loading '${it.nameWithoutExtension}' descriptor from '${it.canonicalPath}'") + it.nameWithoutExtension to parser.parse(it.canonicalPath) as JsonObject + } + .orEmpty() + .toMap() +} + +class InMemoryLibraryResolver(parent: LibraryResolver?, initialCache: Map? = null): LibraryResolver(parent) { + override val cache = hashMapOf() + + init { + initialCache?.forEach { key, value -> + cache[key] = value + } + } + + override fun shouldResolve(reference: LibraryReference): Boolean { + return reference.shouldBeCachedInMemory + } + + override fun tryResolve(reference: LibraryReference): LibraryDescriptor? { + return cache[reference] + } + + override fun save(reference: LibraryReference, descriptor: LibraryDescriptor) { + cache[reference] = descriptor + } } diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/typeProviderTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/typeProviderTests.kt index 387562826..32d6515cf 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/typeProviderTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/typeProviderTests.kt @@ -3,7 +3,13 @@ package org.jetbrains.kotlin.jupyter.test import com.beust.klaxon.JsonObject import com.beust.klaxon.Parser import jupyter.kotlin.receivers.TypeProviderReceiver -import org.jetbrains.kotlin.jupyter.* +import org.jetbrains.kotlin.jupyter.ReplCompilerException +import org.jetbrains.kotlin.jupyter.ReplForJupyterImpl +import org.jetbrains.kotlin.jupyter.ResolverConfig +import org.jetbrains.kotlin.jupyter.defaultRepositories +import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory +import org.jetbrains.kotlin.jupyter.libraries.LibraryResolutionInfo +import org.jetbrains.kotlin.jupyter.libraries.parseLibraryDescriptors import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows @@ -24,7 +30,9 @@ class TypeProviderTests { """.trimIndent() val cp = classpath + File(TypeProviderReceiver::class.java.protectionDomain.codeSource.location.toURI().path) val libJsons = mapOf("mylib" to parser.parse(StringBuilder(descriptor)) as JsonObject) - val repl = ReplForJupyterImpl(cp, ResolverConfig(defaultRepositories, parserLibraryDescriptors(libJsons).asAsync()), TypeProviderReceiver()) + val libraryFactory = LibraryFactory(LibraryResolutionInfo.ByNothing()) + val config = ResolverConfig(defaultRepositories, libraryFactory.getResolverFromNamesMap(parseLibraryDescriptors(libJsons))) + val repl = ReplForJupyterImpl(libraryFactory, cp, null, config, scriptReceivers = listOf(TypeProviderReceiver())) // create list 'l' of size 3 val code1 = """ From e2845fcd154f568b3d6535e99ab854790ad3b1a0 Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Wed, 29 Jul 2020 09:27:48 +0300 Subject: [PATCH 2/7] Pass repl as evaluation parameter on configuration creation --- src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt index 208ef8b0b..584d8b37d 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt @@ -323,7 +323,7 @@ class ReplForJupyterImpl( val scriptClassloader = URLClassLoader(scriptClasspath.map { it.toURI().toURL() }.toTypedArray(), filteringClassLoader) baseClassLoader(scriptClassloader) } - constructorArgs() + constructorArgs(this@ReplForJupyterImpl as KotlinKernelHost) } private var executionCounter = 0 @@ -589,11 +589,7 @@ class ReplForJupyterImpl( is ResultWithDiagnostics.Success -> { val compileResult = compileResultWithDiagnostics.value classWriter?.writeClasses(codeLine, compileResult.get()) - val repl = this - val currentEvalConfig = ScriptEvaluationConfiguration(evaluatorConfiguration) { - constructorArgs.invoke(repl as KotlinKernelHost) - } - val resultWithDiagnostics = runBlocking { evaluator.eval(compileResult, currentEvalConfig) } + val resultWithDiagnostics = runBlocking { evaluator.eval(compileResult, evaluatorConfiguration) } contextUpdater.update() when(resultWithDiagnostics) { From 2d029d8615ab5f4b3d5952f584a110408c81c05c Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Thu, 30 Jul 2020 23:05:19 +0300 Subject: [PATCH 3/7] Update project dictionary --- .idea/dictionaries/ProjectDictionary.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/.idea/dictionaries/ProjectDictionary.xml b/.idea/dictionaries/ProjectDictionary.xml index 84c237075..be051276f 100644 --- a/.idea/dictionaries/ProjectDictionary.xml +++ b/.idea/dictionaries/ProjectDictionary.xml @@ -8,6 +8,7 @@ clikt comms deeplearning + displayname distrib doyaaaaaken dsdsda From 42c0eef116bfe557a2bf36a702754abc9deaabf1 Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Fri, 31 Jul 2020 06:30:57 +0300 Subject: [PATCH 4/7] Add local Maven repository if it exists --- settings.gradle.kts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/settings.gradle.kts b/settings.gradle.kts index 9b4216ef3..7c56670df 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -11,6 +11,12 @@ pluginManagement { // only when using Kotlin EAP releases ... maven { url = uri("https://dl.bintray.com/kotlin/kotlin-eap") } maven { url = uri("https://dl.bintray.com/kotlin/kotlin-dev") } + + // Used for TeamCity build + val m2LocalPath = File(".m2/repository") + if (m2LocalPath.exists()) { + maven(m2LocalPath.toURI()) + } } resolutionStrategy { From f8ff3c3a06012d9e3471be39d3c523e417fd147e Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Mon, 3 Aug 2020 02:21:23 +0300 Subject: [PATCH 5/7] Add default resolution info detection heuristics --- README.md | 7 +++ .../jupyter/libraries/LibraryFactory.kt | 14 +++--- .../libraries/ResolutionInfoProvider.kt | 49 +++++++++++++++++++ .../kotlin/jupyter/libraries/util.kt | 10 ++-- .../org/jetbrains/kotlin/jupyter/magics.kt | 2 +- .../jupyter/test/kernelServerTestsBase.kt | 3 +- .../kotlin/jupyter/test/parseMagicsTests.kt | 5 +- .../kotlin/jupyter/test/replTests.kt | 15 ++++-- .../jetbrains/kotlin/jupyter/test/testUtil.kt | 2 +- .../kotlin/jupyter/test/typeProviderTests.kt | 4 +- 10 files changed, 89 insertions(+), 22 deletions(-) create mode 100644 src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/ResolutionInfoProvider.kt diff --git a/README.md b/README.md index f9dd9edfd..5b954f283 100644 --- a/README.md +++ b/README.md @@ -143,8 +143,15 @@ Other options are resolving library descriptor from a local file or from remote ``` // Load library from file %use mylib@file[/home/user/lib.json] +// Load library from file: kernel will guess it's a file actually +%use @/home/user/libs/lib.json +// Or use another approach: specify a directory and file name without +// extension (it should be JSON in such case) before it +%use lib@/home/user/libs // Load library descriptor from a remote URL %use herlib@url[https://site.com/lib.json] +// If your URL responds with 200(OK), you may skip `url[]` part: +%use @https://site.com/lib.json // You may omit library name for file and URL resolution: %use @file[lib.json] ``` diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryFactory.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryFactory.kt index 67b665b5c..686916984 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryFactory.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/LibraryFactory.kt @@ -7,7 +7,7 @@ import java.net.URL import java.nio.file.Paths class LibraryFactory( - var defaultResolutionInfo: LibraryResolutionInfo, + val resolutionInfoProvider: ResolutionInfoProvider, private val parsers: Map = defaultParsers, ) { fun parseReferenceWithArgs(str: String): Pair> { @@ -25,16 +25,16 @@ class LibraryFactory( private fun parseResolutionInfo(string: String): LibraryResolutionInfo { // In case of empty string after `@`: %use lib@ - if(string.isBlank()) return defaultResolutionInfo + if(string.isBlank()) return resolutionInfoProvider.get() val (type, vars) = parseCall(string, Brackets.SQUARE) - val parser = parsers[type] ?: return LibraryResolutionInfo.getInfoByRef(type) + val parser = parsers[type] ?: return resolutionInfoProvider.get(type) return parser.getInfo(vars) } private fun parseReference(string: String): LibraryReference { val sepIndex = string.indexOf('@') - if (sepIndex == -1) return LibraryReference(defaultResolutionInfo, string) + if (sepIndex == -1) return LibraryReference(resolutionInfoProvider.get(), string) val nameString = string.substring(0, sepIndex) val infoString = string.substring(sepIndex + 1) @@ -43,8 +43,6 @@ class LibraryFactory( } companion object { - fun withDefaultDirectoryResolution(dir: File) = LibraryFactory(LibraryResolutionInfo.ByDir(dir)) - private val defaultParsers = listOf( LibraryResolutionInfoParser.make("ref", listOf(Parameter.Required("ref"))) { args -> LibraryResolutionInfo.getInfoByRef(args["ref"] ?: error("Argument 'ref' should be specified")) @@ -59,5 +57,9 @@ class LibraryFactory( LibraryResolutionInfo.ByURL(URL(args["url"] ?: error("Argument 'url' should be specified"))) }, ).map { it.name to it }.toMap() + + val EMPTY = LibraryFactory(EmptyResolutionInfoProvider) + + fun withDefaultDirectoryResolution(dir: File) = LibraryFactory(StandardResolutionInfoProvider(LibraryResolutionInfo.ByDir(dir))) } } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/ResolutionInfoProvider.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/ResolutionInfoProvider.kt new file mode 100644 index 000000000..fbb9433f2 --- /dev/null +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/ResolutionInfoProvider.kt @@ -0,0 +1,49 @@ +package org.jetbrains.kotlin.jupyter.libraries + +import org.jetbrains.kotlin.jupyter.GitHubApiPrefix +import org.jetbrains.kotlin.jupyter.LibrariesDir +import java.io.File +import java.net.URL + +interface ResolutionInfoProvider { + var fallback: LibraryResolutionInfo + + fun get(): LibraryResolutionInfo = fallback + fun get(string: String): LibraryResolutionInfo +} + +object EmptyResolutionInfoProvider : ResolutionInfoProvider { + private val fallbackInfo = LibraryResolutionInfo.ByNothing() + + override var fallback: LibraryResolutionInfo + get() = fallbackInfo + set(_) {} + + override fun get(string: String) = LibraryResolutionInfo.getInfoByRef(string) +} + +class StandardResolutionInfoProvider(override var fallback: LibraryResolutionInfo) : ResolutionInfoProvider { + override fun get(string: String): LibraryResolutionInfo { + return tryGetAsRef(string) ?: tryGetAsDir(string) ?: tryGetAsFile(string) ?: tryGetAsURL(string) ?: fallback + } + + private fun tryGetAsRef(ref: String): LibraryResolutionInfo? { + val response = khttp.get("$GitHubApiPrefix/contents/$LibrariesDir?ref=$ref") + return if (response.statusCode == 200) LibraryResolutionInfo.getInfoByRef(ref) else null + } + + private fun tryGetAsDir(dirName: String): LibraryResolutionInfo? { + val file = File(dirName) + return if (file.isDirectory) LibraryResolutionInfo.ByDir(file) else null + } + + private fun tryGetAsFile(fileName: String): LibraryResolutionInfo? { + val file = File(fileName) + return if (file.isFile) LibraryResolutionInfo.ByFile(file) else null + } + + private fun tryGetAsURL(url: String): LibraryResolutionInfo? { + val response = khttp.get(url) + return if (response.statusCode == 200) LibraryResolutionInfo.ByURL(URL(url)) else null + } +} diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/util.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/util.kt index 09f585171..8bae0c1aa 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/util.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries/util.kt @@ -32,21 +32,21 @@ enum class DefaultInfoSwitch { GIT_REFERENCE, DIRECTORY } -class LibraryFactoryDefaultInfoSwitcher(private val libraryFactory: LibraryFactory, initialSwitchVal: T, private val switcher: (T) -> LibraryResolutionInfo) { +class LibraryFactoryDefaultInfoSwitcher(private val infoProvider: ResolutionInfoProvider, initialSwitchVal: T, private val switcher: (T) -> LibraryResolutionInfo) { private val defaultInfoCache = hashMapOf() var switch: T = initialSwitchVal set(value) { - libraryFactory.defaultResolutionInfo = defaultInfoCache.getOrPut(value) { switcher(value) } + infoProvider.fallback = defaultInfoCache.getOrPut(value) { switcher(value) } field = value } companion object { - fun default(factory: LibraryFactory, defaultDir: File, defaultRef: String): LibraryFactoryDefaultInfoSwitcher { - val initialInfo = factory.defaultResolutionInfo + fun default(provider: ResolutionInfoProvider, defaultDir: File, defaultRef: String): LibraryFactoryDefaultInfoSwitcher { + val initialInfo = provider.fallback val dirInfo = if (initialInfo is LibraryResolutionInfo.ByDir) initialInfo else LibraryResolutionInfo.ByDir(defaultDir) val refInfo = if (initialInfo is LibraryResolutionInfo.ByGitRef) initialInfo else LibraryResolutionInfo.getInfoByRef(defaultRef) - return LibraryFactoryDefaultInfoSwitcher(factory, DefaultInfoSwitch.DIRECTORY) { switch -> + return LibraryFactoryDefaultInfoSwitcher(provider, DefaultInfoSwitch.DIRECTORY) { switch -> when(switch) { DefaultInfoSwitch.DIRECTORY -> dirInfo DefaultInfoSwitch.GIT_REFERENCE -> refInfo diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt index 8ee333dc1..4fef03221 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt @@ -33,7 +33,7 @@ data class MagicProcessingResult(val code: String, val libraries: List): OutputConfig { diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/kernelServerTestsBase.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/kernelServerTestsBase.kt index 1d9a610fe..453a57753 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/kernelServerTestsBase.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/kernelServerTestsBase.kt @@ -8,6 +8,7 @@ import org.jetbrains.kotlin.jupyter.Message import org.jetbrains.kotlin.jupyter.defaultRuntimeProperties import org.jetbrains.kotlin.jupyter.iKotlinClass import org.jetbrains.kotlin.jupyter.kernelServer +import org.jetbrains.kotlin.jupyter.libraries.EmptyResolutionInfoProvider import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory import org.jetbrains.kotlin.jupyter.libraries.LibraryResolutionInfo import org.jetbrains.kotlin.jupyter.makeHeader @@ -37,7 +38,7 @@ open class KernelServerTestsBase { scriptClasspath = classpath, resolverConfig = null, homeDir = File(""), - libraryFactory = LibraryFactory(LibraryResolutionInfo.ByNothing()) + libraryFactory = LibraryFactory.EMPTY ) private val sessionId = UUID.randomUUID().toString() diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseMagicsTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseMagicsTests.kt index 2acda029b..41c71c8ae 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseMagicsTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/parseMagicsTests.kt @@ -8,6 +8,7 @@ import org.jetbrains.kotlin.jupyter.MagicsProcessor import org.jetbrains.kotlin.jupyter.OutputConfig import org.jetbrains.kotlin.jupyter.ReplOptions import org.jetbrains.kotlin.jupyter.defaultRuntimeProperties +import org.jetbrains.kotlin.jupyter.libraries.EmptyResolutionInfoProvider import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory import org.jetbrains.kotlin.jupyter.libraries.LibraryResolutionInfo import org.jetbrains.kotlin.jupyter.repl.SourceCodeImpl @@ -19,7 +20,7 @@ import java.io.File import kotlin.test.assertTrue class ParseArgumentsTests { - private val libraryFactory = LibraryFactory(LibraryResolutionInfo.ByNothing()) + private val libraryFactory = LibraryFactory.EMPTY @Test fun test1() { @@ -101,7 +102,7 @@ class ParseMagicsTests { private val options = TestReplOptions() private fun test(code: String, expectedProcessedCode: String, librariesChecker: (List) -> Unit = {}) { - val libraryFactory = LibraryFactory(LibraryResolutionInfo.ByNothing()) + val libraryFactory = LibraryFactory.EMPTY val processor = MagicsProcessor(options, LibrariesProcessor(libraryFactory.testResolverConfig.libraries, defaultRuntimeProperties, libraryFactory)) with(processor.processMagics(code, tryIgnoreErrors = true)) { assertEquals(expectedProcessedCode, this.code) diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt index 753eee13b..653f12433 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt @@ -43,7 +43,7 @@ abstract class AbstractReplTest { companion object { @JvmStatic - protected val libraryFactory = LibraryFactory(LibraryResolutionInfo.ByNothing()) + protected val libraryFactory = LibraryFactory.EMPTY @JvmStatic protected val homeDir = File("") @@ -569,15 +569,16 @@ class ReplWithResolverTest : AbstractReplTest() { @Test fun testDefaultInfoSwitcher() { val repl = getReplWithStandardResolver() + val infoProvider = repl.libraryFactory.resolutionInfoProvider - val initialDefaultResolutionInfo = repl.libraryFactory.defaultResolutionInfo + val initialDefaultResolutionInfo = infoProvider.fallback assertTrue(initialDefaultResolutionInfo is LibraryResolutionInfo.ByDir) repl.eval("%useLatestDescriptors") - assertTrue(repl.libraryFactory.defaultResolutionInfo is LibraryResolutionInfo.ByGitRef) + assertTrue(infoProvider.fallback is LibraryResolutionInfo.ByGitRef) repl.eval("%useLatestDescriptors -off") - assertTrue(repl.libraryFactory.defaultResolutionInfo === initialDefaultResolutionInfo) + assertTrue(infoProvider.fallback === initialDefaultResolutionInfo) } @Test @@ -604,6 +605,12 @@ class ReplWithResolverTest : AbstractReplTest() { assertEquals(1, displays.count()) assertNull(res3.resultValue) displays.clear() + + val res4 = repl.eval(""" + %use @$libraryPath(name=z, value=44) + z + """.trimIndent()) + assertEquals(44, res4.resultValue) } @Test diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/testUtil.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/testUtil.kt index 357d060bd..867aa70ef 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/testUtil.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/testUtil.kt @@ -46,7 +46,7 @@ fun Collection>.toLibraries(libraryFactory: LibraryFactory) } fun LibraryFactory.getResolverFromNamesMap(map: Map): LibraryResolver { - return InMemoryLibraryResolver(null, map.mapKeys { entry -> LibraryReference(defaultResolutionInfo, entry.key) }) + return InMemoryLibraryResolver(null, map.mapKeys { entry -> LibraryReference(resolutionInfoProvider.get(), entry.key) }) } fun readLibraries(basePath: String? = null): Map { diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/typeProviderTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/typeProviderTests.kt index 32d6515cf..659e699ed 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/typeProviderTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/typeProviderTests.kt @@ -7,8 +7,8 @@ import org.jetbrains.kotlin.jupyter.ReplCompilerException import org.jetbrains.kotlin.jupyter.ReplForJupyterImpl import org.jetbrains.kotlin.jupyter.ResolverConfig import org.jetbrains.kotlin.jupyter.defaultRepositories +import org.jetbrains.kotlin.jupyter.libraries.EmptyResolutionInfoProvider import org.jetbrains.kotlin.jupyter.libraries.LibraryFactory -import org.jetbrains.kotlin.jupyter.libraries.LibraryResolutionInfo import org.jetbrains.kotlin.jupyter.libraries.parseLibraryDescriptors import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -30,7 +30,7 @@ class TypeProviderTests { """.trimIndent() val cp = classpath + File(TypeProviderReceiver::class.java.protectionDomain.codeSource.location.toURI().path) val libJsons = mapOf("mylib" to parser.parse(StringBuilder(descriptor)) as JsonObject) - val libraryFactory = LibraryFactory(LibraryResolutionInfo.ByNothing()) + val libraryFactory = LibraryFactory.EMPTY val config = ResolverConfig(defaultRepositories, libraryFactory.getResolverFromNamesMap(parseLibraryDescriptors(libJsons))) val repl = ReplForJupyterImpl(libraryFactory, cp, null, config, scriptReceivers = listOf(TypeProviderReceiver())) From 252a94acc3eb298145813a07989e42eca8eba110 Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Tue, 18 Aug 2020 16:20:35 +0300 Subject: [PATCH 6/7] Upgrade Kotlin version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index aa5499f97..7570c7bd6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ # kotlinVersion=1.4.255-SNAPSHOT -kotlinVersion=1.4.20-dev-2342 +kotlinVersion=1.4.20-dev-3647 kotlinLanguageLevel=1.4 jvmTarget=1.8 From 528d6caa2b929dfc9320c9ea415e47f1bd2a7bd4 Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Tue, 18 Aug 2020 16:42:43 +0300 Subject: [PATCH 7/7] Fix commit hash in test --- src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt index 653f12433..69c50d236 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt @@ -585,7 +585,7 @@ class ReplWithResolverTest : AbstractReplTest() { fun testUseFileUrlRef() { val repl = getReplWithStandardResolver() - val commit = "1f56d74a88f6fb78306d685d0b3aaf07113a8abf" + val commit = "561ce1a324a9434d3481456b11678851b48a3132" val libraryPath = "src/test/testData/test-init.json" val res1 = repl.eval("""