Skip to content

Commit

Permalink
Add type names filtering for integration classes
Browse files Browse the repository at this point in the history
Fixes #363
  • Loading branch information
ileasile committed Apr 29, 2022
1 parent f5df30d commit b4d6610
Show file tree
Hide file tree
Showing 17 changed files with 352 additions and 17 deletions.
3 changes: 2 additions & 1 deletion docs/libraries.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ Library descriptor is a `<libName>.json` file with the following fields:
- `renderers`: a mapping from fully qualified names of types to be rendered to the Kotlin expression returning output value.
Source object is referenced as `$it`
- `resources`: a list of JS/CSS resources. See [this descriptor](../src/test/testData/lib-with-resources.json) for example
- `integrationTypeNameRules`: a list of rules for integration classes which are about to be loaded transitively. Each rule has the form `[+|-]:<pattern>` where `+` or `-` denotes if this pattern is accepted or declined. Pattern may consist of any characters. Special combinations are allowed: `?` (any single character or no character), `*` (any character excluding dot), `**` (any character).

*All fields are optional

Expand Down Expand Up @@ -71,7 +72,7 @@ plugins {
This plugin adds following dependencies to your project:

| Artifact | Gradle option to exclude/include | Enabled by default | Dependency scope | Method for adding dependency manually |
| :------------------------------- | :------------------------------- | :----------------- | :------------------- | :--------------------------------------- |
|:---------------------------------|:---------------------------------|:-------------------|:---------------------|:-----------------------------------------|
| `kotlin-jupyter-api` | `kotlin.jupyter.add.api` | yes | `compileOnly` | `addApiDependency(version: String?)` |
| `kotlin-jupyter-api-annotations` | `kotlin.jupyter.add.scanner` | no | `compileOnly` | `addScannerDependency(version: String?)` |
| `kotlin-jupyter-test-kit` | `kotlin.jupyter.add.testkit` | yes | `testImplementation` | `addTestKitDependency(version: String?)` |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ interface KotlinKernelHost {
*/
fun addLibraries(libraries: Collection<LibraryDefinition>)

/**
* Says whether this [typeName] should be loaded as integration based on loaded libraries.
* `null` means that loaded libraries don't care about this [typeName].
*/
fun acceptsIntegrationTypeName(typeName: String): Boolean?

/**
* Loads Kotlin standard artifacts (org.jetbrains.kotlin:kotlin-$name:$version)
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,11 @@ import org.jetbrains.kotlinx.jupyter.api.ResultHandlerExecution
import org.jetbrains.kotlinx.jupyter.api.SubtypeRendererTypeHandler
import org.jetbrains.kotlinx.jupyter.api.SubtypeThrowableRenderer
import org.jetbrains.kotlinx.jupyter.api.ThrowableRenderer
import org.jetbrains.kotlinx.jupyter.api.TypeName
import org.jetbrains.kotlinx.jupyter.api.VariableDeclarationCallback
import org.jetbrains.kotlinx.jupyter.api.VariableUpdateCallback
import org.jetbrains.kotlinx.jupyter.util.AcceptanceRule
import org.jetbrains.kotlinx.jupyter.util.NameAcceptanceRule
import kotlin.reflect.KMutableProperty

/**
Expand Down Expand Up @@ -66,6 +69,8 @@ abstract class JupyterIntegration : LibraryDefinitionProducer {

private val internalVariablesMarkers = mutableListOf<InternalVariablesMarker>()

private val integrationTypeNameRules = mutableListOf<AcceptanceRule<String>>()

fun addRenderer(handler: RendererHandler) {
renderers.add(handler)
}
Expand Down Expand Up @@ -197,6 +202,31 @@ abstract class JupyterIntegration : LibraryDefinitionProducer {
preprocessCodeWithLibraries { CodePreprocessor.Result(this.callback(it)) }
}

/**
* All integrations transitively loaded by this integration will be tested against
* passed acceptance rule and won't be loaded if the rule returned `false`.
* If there were no acceptance rules that returned not-null values, integration
* **will be loaded**. If there are several acceptance rules that returned not-null values,
* the latest one will be taken into account.
*/
fun addIntegrationTypeNameRule(rule: AcceptanceRule<TypeName>) {
integrationTypeNameRules.add(rule)
}

/**
* See [addIntegrationTypeNameRule]
*/
fun acceptIntegrationTypeNameIf(predicate: (TypeName) -> Boolean) {
addIntegrationTypeNameRule(NameAcceptanceRule(true, predicate))
}

/**
* See [addIntegrationTypeNameRule]
*/
fun discardIntegrationTypeNameIf(predicate: (TypeName) -> Boolean) {
addIntegrationTypeNameRule(NameAcceptanceRule(false, predicate))
}

internal fun getDefinition() =
libraryDefinition {
it.init = init
Expand All @@ -214,6 +244,7 @@ abstract class JupyterIntegration : LibraryDefinitionProducer {
it.resources = resources
it.codePreprocessors = codePreprocessors
it.internalVariablesMarkers = internalVariablesMarkers
it.integrationTypeNameRules = integrationTypeNameRules
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import org.jetbrains.kotlinx.jupyter.api.InternalVariablesMarker
import org.jetbrains.kotlinx.jupyter.api.KotlinKernelVersion
import org.jetbrains.kotlinx.jupyter.api.RendererHandler
import org.jetbrains.kotlinx.jupyter.api.ThrowableRenderer
import org.jetbrains.kotlinx.jupyter.util.AcceptanceRule

/**
* Library definition represents "library" concept in Kotlin kernel.
Expand Down Expand Up @@ -117,4 +118,10 @@ interface LibraryDefinition {
*/
val internalVariablesMarkers: List<InternalVariablesMarker>
get() = emptyList()

/**
* Integration type name rules for the library integration classes which are about to be loaded transitively
*/
val integrationTypeNameRules: List<AcceptanceRule<String>>
get() = emptyList()
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import org.jetbrains.kotlinx.jupyter.api.InternalVariablesMarker
import org.jetbrains.kotlinx.jupyter.api.KotlinKernelVersion
import org.jetbrains.kotlinx.jupyter.api.RendererHandler
import org.jetbrains.kotlinx.jupyter.api.ThrowableRenderer
import org.jetbrains.kotlinx.jupyter.util.AcceptanceRule

/**
* Trivial implementation of [LibraryDefinition] - simple container.
Expand All @@ -32,6 +33,7 @@ class LibraryDefinitionImpl private constructor() : LibraryDefinition {
override var internalVariablesMarkers: List<InternalVariablesMarker> = emptyList()
override var minKernelVersion: KotlinKernelVersion? = null
override var originalDescriptorText: String? = null
override var integrationTypeNameRules: List<AcceptanceRule<String>> = emptyList()

companion object {
internal fun build(buildAction: (LibraryDefinitionImpl) -> Unit): LibraryDefinition {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package org.jetbrains.kotlinx.jupyter.util

import kotlinx.serialization.Serializable
import org.jetbrains.kotlinx.jupyter.api.TypeName
import org.jetbrains.kotlinx.jupyter.api.libraries.VariablesSubstitutionAware

/**
* Acceptance rule either says it accepts object or delegates it to some other rule returning null
*/
fun interface AcceptanceRule<T> {
fun accepts(obj: T): Boolean?
}

/**
* Acceptance rule that has only two answers: yes/no (depending on the [acceptsFlag]) and "don't know"
*/
interface FlagAcceptanceRule<T> : AcceptanceRule<T> {
val acceptsFlag: Boolean
fun appliesTo(obj: T): Boolean

override fun accepts(obj: T): Boolean? {
return if (appliesTo(obj)) acceptsFlag else null
}
}

/**
* Acceptance rule for type names
*/
class NameAcceptanceRule(
override val acceptsFlag: Boolean,
private val appliesPredicate: (TypeName) -> Boolean
) : FlagAcceptanceRule<TypeName> {
override fun appliesTo(obj: TypeName): Boolean {
return appliesPredicate(obj)
}
}

/**
* Acceptance rule for type names based on [pattern].
* Pattern may consist of any characters and of 3 special combinations:
* 1) `?` - any single character or no character
* 2) `*` - any character sequence excluding dot (`.`)
* 3) `**` - any character sequence
*
* For example, pattern `org.jetbrains.kotlin?.**.jupyter.*` matches following names:
* - `org.jetbrains.kotlin.my.package.jupyter.Integration`
* - `org.jetbrains.kotlinx.some_package.jupyter.SomeClass`
*
* It doesn't match name `org.jetbrains.kotlin.my.package.jupyter.integration.MyClass`
*/
@Serializable(PatternNameAcceptanceRuleSerializer::class)
class PatternNameAcceptanceRule(
override val acceptsFlag: Boolean,
val pattern: String,
) : FlagAcceptanceRule<TypeName>, VariablesSubstitutionAware<PatternNameAcceptanceRule> {
private val regex by lazy {
buildString {
var i = 0
while (i < pattern.length) {
val c = pattern[i]
val nextC = pattern.getOrNull(i + 1)

when (c) {
'.' -> append("\\.")
'*' -> when (nextC) {
'*' -> {
append(".*")
++i
}
else -> append("[^.]*")
}
'?' -> append(".?")
'[', ']', '(', ')', '{', '}', '\\', '$', '^', '+', '|' -> {
append('\\')
append(c)
}
else -> append(c)
}
++i
}
}.toRegex()
}

override fun appliesTo(obj: TypeName): Boolean {
return regex.matches(obj)
}

override fun replaceVariables(mapping: Map<String, String>): PatternNameAcceptanceRule {
val newPattern = replaceVariables(pattern, mapping)
if (pattern == newPattern) return this
return PatternNameAcceptanceRule(acceptsFlag, newPattern)
}
}

/**
* List of acceptance rules:
* 1) accepts [obj] if latest not-null acceptance result is `true`
* 2) doesn't accept [obj] if latest not-null acceptance result is `false`
* 3) returns `null` if all acceptance results are `null` or the iterable is empty
*/
fun <T> Iterable<AcceptanceRule<T>>.accepts(obj: T): Boolean? {
return mapNotNull { it.accepts(obj) }.lastOrNull()
}
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ abstract class ListToMapSerializer<T, K, V>(
}

override fun serialize(encoder: Encoder, value: List<T>) {
val tempMap = value.map(reverseMapper).toMap()
val tempMap = value.associate(reverseMapper)
utilSerializer.serialize(encoder, tempMap)
}
}
Expand Down Expand Up @@ -115,3 +115,30 @@ object ResourceBunchSerializer : KSerializer<ResourceFallbacksBundle> {
encoder.encodeSerializableValue(serializer(), value.locations)
}
}

object PatternNameAcceptanceRuleSerializer : KSerializer<PatternNameAcceptanceRule> {
override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor(PatternNameAcceptanceRule::class.qualifiedName!!, PrimitiveKind.STRING)

override fun deserialize(decoder: Decoder): PatternNameAcceptanceRule {
val rule = decoder.decodeString()
fun throwError(): Nothing = throw SerializationException("Wrong format of pattern rule: $rule")

val parts = rule.split(':').map { it.trim() }
val (sign, pattern) = when (parts.size) {
1 -> "+" to parts[0]
2 -> parts[0] to parts[1]
else -> throwError()
}
val accepts = when (sign) {
"+" -> true
"-" -> false
else -> throwError()
}

return PatternNameAcceptanceRule(accepts, pattern)
}

override fun serialize(encoder: Encoder, value: PatternNameAcceptanceRule) {
encoder.encodeString("${ if (value.acceptsFlag) '+' else '-' }:${value.pattern}")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import org.jetbrains.kotlinx.jupyter.api.libraries.libraryDefinition
import org.jetbrains.kotlinx.jupyter.config.currentKernelVersion
import org.jetbrains.kotlinx.jupyter.exceptions.ReplPreprocessingException
import org.jetbrains.kotlinx.jupyter.util.KotlinKernelVersionSerializer
import org.jetbrains.kotlinx.jupyter.util.PatternNameAcceptanceRule
import org.jetbrains.kotlinx.jupyter.util.RenderersSerializer
import org.jetbrains.kotlinx.jupyter.util.replaceVariables

Expand Down Expand Up @@ -42,6 +43,8 @@ class LibraryDescriptor(

@Serializable(KotlinKernelVersionSerializer::class)
val minKernelVersion: KotlinKernelVersion? = null,

val integrationTypeNameRules: List<PatternNameAcceptanceRule> = emptyList(),
) {
fun convertToDefinition(arguments: List<Variable>): LibraryDefinition {
val mapping = substituteArguments(variables, arguments)
Expand Down Expand Up @@ -90,6 +93,7 @@ class LibraryDescriptor(
it.renderers = renderers.replaceVariables(mapping)
it.resources = resources.replaceVariables(mapping)
it.minKernelVersion = minKernelVersion
it.integrationTypeNameRules = integrationTypeNameRules.replaceVariables(mapping)
it.originalDescriptorText = Json.encodeToString(this)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,33 @@ import org.jetbrains.kotlinx.jupyter.exceptions.ReplException

class LibrariesScanner(val notebook: Notebook) {
private val processedFQNs = mutableSetOf<TypeName>()

private fun <T, I : LibrariesInstantiable<T>> Iterable<I>.filterProcessed(): List<I> {
return filter { processedFQNs.add(it.fqn) }
private val discardedFQNs = mutableSetOf<TypeName>()

private fun <T, I : LibrariesInstantiable<T>> Iterable<I>.filterNamesToLoad(host: KotlinKernelHost): List<I> {
return filter {
val typeName = it.fqn
val acceptance = host.acceptsIntegrationTypeName(typeName)
log.debug("Acceptance result for $typeName: $acceptance")
when (acceptance) {
true -> processedFQNs.add(typeName)
false -> {
discardedFQNs.add(typeName)
false
}
null -> typeName !in discardedFQNs && processedFQNs.add(typeName)
}
}
}

fun addLibrariesFromClassLoader(classLoader: ClassLoader, host: KotlinKernelHost) {
val scanResult = scanForLibraries(classLoader)
val scanResult = scanForLibraries(classLoader, host)
log.debug("Scanning for libraries is done. Detected FQNs: ${Json.encodeToString(scanResult)}")
val libraries = instantiateLibraries(classLoader, scanResult, notebook)
log.debug("Number of detected definitions: ${libraries.size}")
host.addLibraries(libraries)
}

private fun scanForLibraries(classLoader: ClassLoader): LibrariesScanResult {
private fun scanForLibraries(classLoader: ClassLoader, host: KotlinKernelHost): LibrariesScanResult {
val results = classLoader.getResources("$KOTLIN_JUPYTER_RESOURCES_PATH/$KOTLIN_JUPYTER_LIBRARIES_FILE_NAME").toList().map { url ->
val contents = url.readText()
Json.decodeFromString<LibrariesScanResult>(contents)
Expand All @@ -47,8 +60,8 @@ class LibrariesScanner(val notebook: Notebook) {
}

return LibrariesScanResult(
definitions.filterProcessed(),
producers.filterProcessed(),
definitions.filterNamesToLoad(host),
producers.filterNamesToLoad(host),
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package org.jetbrains.kotlinx.jupyter.repl
import org.jetbrains.kotlinx.jupyter.api.Code
import org.jetbrains.kotlinx.jupyter.api.libraries.ExecutionHost
import org.jetbrains.kotlinx.jupyter.messaging.DisplayHandler
import org.jetbrains.kotlinx.jupyter.repl.impl.ExecutionStackFrame

/**
* Executes notebook cell code.
Expand All @@ -18,7 +19,8 @@ interface CellExecutor : ExecutionHost {
processMagics: Boolean = true,
invokeAfterCallbacks: Boolean = true,
currentCellId: Int = -1,
callback: ExecutionStartedCallback? = null
stackFrame: ExecutionStackFrame? = null,
callback: ExecutionStartedCallback? = null,
): InternalEvalResult
}

Expand Down
Loading

0 comments on commit b4d6610

Please sign in to comment.