Refactor libraries handling, add handling of descriptors from different sources

README.md

@@ -100,6 +100,7 @@ The following line magics are supported:
  - `%use <lib1>, <lib2> ...` - injects code for supported libraries: artifact resolution, default imports, initialization code, type renderers
  - `%trackClasspath` - logs any changes of current classpath. Useful for debugging artifact resolution failures
  - `%trackExecution` - logs pieces of code that are going to be executed. Useful for debugging of libraries support
+ - `%useLatestDescriptors` - use latest versions of library descriptors available. By default, bundled descriptors are used
  - `%output [options]` - output capturing settings.
 
  See detailed info about line magics [here](doc/magics.md).
@@ -128,6 +129,32 @@ Several libraries can be included in single `%use` statement, separated by `,`:
 ```
 %use lets-plot, krangl, mysql(8.0.15)
 ```
+You can also specify the source of library descriptor. By default, it's downloaded from the latest commit on the
+branch which kernel was built from. If you want to try descriptor from another revision, use the following syntax:
+```
+// Specify tag
+%use lets-plot@0.8.2.5
+// Specify commit sha, with more verbose syntax
+%use lets-plot@ref[24a040fe22335648885b106e2f4ddd63b4d49469]
+// Specify git ref along with library arguments
+%use krangl@dev(0.10)
+```
+Other options are resolving library descriptor from a local file or from remote URL:
+```
+// Load library from file
+%use mylib@file[/home/user/lib.json]
+// Load library from file: kernel will guess it's a file actually
+%use @/home/user/libs/lib.json
+// Or use another approach: specify a directory and file name without 
+// extension (it should be JSON in such case) before it
+%use lib@/home/user/libs
+// Load library descriptor from a remote URL
+%use herlib@url[https://site.com/lib.json]
+// If your URL responds with 200(OK), you may skip `url[]` part:
+%use @https://site.com/lib.json
+// You may omit library name for file and URL resolution:
+%use @file[lib.json]
+```
 
 List of supported libraries:
  - [klaxon](https://github.com/cbeust/klaxon) - JSON parser for Kotlin

gradle.properties

@@ -1,5 +1,5 @@
 # kotlinVersion=1.4.255-SNAPSHOT
-kotlinVersion=1.4.20-dev-2342
+kotlinVersion=1.4.20-dev-3647
 kotlinLanguageLevel=1.4
 jvmTarget=1.8
 

settings.gradle.kts

@@ -11,6 +11,12 @@ pluginManagement {
         // only when using Kotlin EAP releases ...
         maven { url = uri("https://dl.bintray.com/kotlin/kotlin-eap") }
         maven { url = uri("https://dl.bintray.com/kotlin/kotlin-dev") }
+
+        // Used for TeamCity build
+        val m2LocalPath = File(".m2/repository")
+        if (m2LocalPath.exists()) {
+            maven(m2LocalPath.toURI())
+        }
     }
 
     resolutionStrategy {

src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt

@@ -1,7 +1,7 @@
 package org.jetbrains.kotlin.jupyter
 
 import jupyter.kotlin.KotlinFunctionInfo
-import org.jetbrains.kotlin.jupyter.repl.reflect.ContextUpdater
+import org.jetbrains.kotlin.jupyter.repl.ContextUpdater
 
 interface AnnotationsProcessor {
 
@@ -57,4 +57,4 @@ class AnnotationsProcessorImpl(private val contextUpdater: ContextUpdater) : Ann
         }
         return codeToExecute
     }
-}
+}

src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt

@@ -1,10 +1,11 @@
 package org.jetbrains.kotlin.jupyter
 
 import jupyter.kotlin.textResult
-import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResult
-import org.jetbrains.kotlin.jupyter.repl.completion.KotlinCompleter
-import org.jetbrains.kotlin.jupyter.repl.completion.ListErrorsResult
-import org.jetbrains.kotlin.jupyter.repl.completion.SourceCodeImpl
+import org.jetbrains.kotlin.jupyter.libraries.parseLibraryDescriptor
+import org.jetbrains.kotlin.jupyter.repl.CompletionResult
+import org.jetbrains.kotlin.jupyter.repl.KotlinCompleter
+import org.jetbrains.kotlin.jupyter.repl.ListErrorsResult
+import org.jetbrains.kotlin.jupyter.repl.SourceCodeImpl
 import kotlin.script.experimental.api.ScriptDiagnostic
 import kotlin.script.experimental.api.SourceCode
 import kotlin.script.experimental.api.SourceCodeCompletionVariant
@@ -51,12 +52,12 @@ fun doCommandCompletion(code: String, cursor: Int): CompletionResult {
     return KotlinCompleter.getResult(code, cursor, completions)
 }
 
-fun runCommand(code: String, repl: ReplForJupyter?): Response {
+fun runCommand(code: String, repl: ReplForJupyter): Response {
     val args = code.trim().substring(1).split(" ")
     val cmd = getCommand(args[0]) ?: return AbortResponseWithMessage(textResult("Failed!"), "unknown command: $code\nto see available commands, enter :help")
     return when (cmd) {
         ReplCommands.classpath -> {
-            val cp = repl!!.currentClasspath
+            val cp = repl.currentClasspath
             OkResponseWithMessage(textResult("Current classpath (${cp.count()} paths):\n${cp.joinToString("\n")}"))
         }
         ReplCommands.help -> {
@@ -66,9 +67,16 @@ fun runCommand(code: String, repl: ReplForJupyter?): Response {
                 if (it.argumentsUsage != null) s += "\n        Usage: %${it.name} ${it.argumentsUsage}"
                 s
             }
-            val libraries = repl?.resolverConfig?.libraries?.awaitBlocking()?.toList()?.joinToStringIndented {
-                "${it.first} ${it.second.link ?: ""}"
-            }
+            val libraryFiles =
+                    repl.homeDir?.resolve(LibrariesDir)?.listFiles { file -> file.isFile && file.name.endsWith(".$LibraryDescriptorExt") } ?: emptyArray()
+            val libraries = libraryFiles.toList().mapNotNull { file ->
+                val libraryName = file.nameWithoutExtension
+                log.info("Parsing descriptor for library '$libraryName'")
+                val descriptor = log.catchAll("Parsing descriptor for library '$libraryName' failed") {
+                    parseLibraryDescriptor(file.readText())
+                }
+                if (descriptor != null) "$libraryName ${descriptor.link ?: ""}" else null
+            }.joinToStringIndented()
             OkResponseWithMessage(textResult("Commands:\n$commands\n\nMagics\n$magics\n\nSupported libraries:\n$libraries"))
         }
     }