Skip to content

Commit

Permalink
Fix merge, include tokens in Lionweb parsing results
Browse files Browse the repository at this point in the history
  • Loading branch information
@alessiostalla
alessiostalla committed Aug 14, 2024
1 parent 233fbe6 commit c43cae3
Show file tree
Hide file tree
Showing 7 changed files with 113 additions and 36 deletions.
18 changes: 11 additions & 7 deletions core/src/main/kotlin/com/strumenta/kolasu/parsing/KolasuParser.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,12 @@ interface TokenFactory<T : KolasuToken> {
fun categoryOf(t: Token): TokenCategory = TokenCategory.PLAIN_TEXT

fun convertToken(t: Token): T
}

private fun convertToken(terminalNode: TerminalNode): T = convertToken(terminalNode.symbol)
open class ANTLRTokenFactory : TokenFactory<KolasuANTLRToken> {
override fun convertToken(t: Token): KolasuANTLRToken = KolasuANTLRToken(categoryOf(t), t)

fun extractTokens(result: ParsingResult<*>): LexingResult<T>? {
fun extractTokens(result: ParsingResult<*>): LexingResult<KolasuANTLRToken>? {
val antlrTerminals = mutableListOf<TerminalNode>()
fun extractTokensFromParseTree(pt: ParseTree?) {
if (pt is TerminalNode) {
Expand All @@ -49,20 +51,22 @@ interface TokenFactory<T : KolasuToken> {
}
}

val ptRoot = result.firstStage?.root
val ptRoot = result.firstStage?.root ?: if (result.root?.origin is ParseTreeOrigin) {
(result.root.origin as ParseTreeOrigin).parseTree
} else {
null
}
return if (ptRoot != null) {
extractTokensFromParseTree(ptRoot)
antlrTerminals.sortBy { it.symbol.tokenIndex }
val tokens = antlrTerminals.map { convertToken(it) }.toMutableList()
LexingResult(result.issues, tokens, result.code, result.firstStage.lexingTime, result.source)
LexingResult(result.issues, tokens, result.code, result.firstStage?.lexingTime, result.source)
} else {
null
}
}
}

open class ANTLRTokenFactory : TokenFactory<KolasuANTLRToken> {
override fun convertToken(t: Token): KolasuANTLRToken = KolasuANTLRToken(categoryOf(t), t)
private fun convertToken(terminalNode: TerminalNode): KolasuANTLRToken = convertToken(terminalNode.symbol)
}

abstract class KolasuANTLRLexer<T : KolasuToken>(val tokenFactory: TokenFactory<T>) : KolasuLexer<T> {
Expand Down
4 changes: 2 additions & 2 deletions core/src/main/kotlin/com/strumenta/kolasu/parsing/Parsing.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ data class TokenCategory(val type: String) {
open class KolasuToken(
open val category: TokenCategory,
open val position: Position,
open val text: String
open val text: String? = null
) : Serializable

/**
Expand Down Expand Up @@ -155,7 +155,7 @@ class FirstStageParsingResult<C : ParserRuleContext>(
* @param firstStage the result of the first parsing stage (from source code to parse tree).
* @param time the time spent in the entire parsing process.
*/
class ParsingResult<RootNode : Node>(
open class ParsingResult<RootNode : Node>(
issues: List<Issue>,
val root: RootNode?,
code: String? = null,
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/kotlin/com/strumenta/kolasu/parsing/KolasuParserTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class KolasuParserTest {
""".trimMargin()
)
assertNotNull(result)
val lexingResult = parser.tokenFactory.extractTokens(result)
val lexingResult = (parser.tokenFactory as ANTLRTokenFactory).extractTokens(result)
assertNotNull(lexingResult)
assertEquals(11, lexingResult.tokens.size)
val text = lexingResult.tokens.map { it.text }
Expand Down
59 changes: 38 additions & 21 deletions lionweb/src/main/kotlin/com/strumenta/kolasu/lionweb/LionWebModelConverter.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import com.strumenta.kolasu.model.isAttribute
import com.strumenta.kolasu.model.isContainment
import com.strumenta.kolasu.model.isReference
import com.strumenta.kolasu.model.nodeOriginalProperties
import com.strumenta.kolasu.parsing.FirstStageParsingResult
import com.strumenta.kolasu.parsing.KolasuToken
import com.strumenta.kolasu.parsing.ParsingResult
import com.strumenta.kolasu.transformation.MissingASTTransformation
import com.strumenta.kolasu.traversing.walk
Expand Down Expand Up @@ -83,6 +85,15 @@ class LionWebModelConverter(
initialLanguageConverter: LionWebLanguageConverter = LionWebLanguageConverter()
) {
companion object {
private val kFeaturesCache = mutableMapOf<Class<*>, Map<String, Feature>>()
private val lwFeaturesCache = mutableMapOf<Classifier<*>, Map<String, LWFeature<*>>>()

fun lwFeatureByName(classifier: Classifier<*>, featureName: String): LWFeature<*>? {
return lwFeaturesCache.getOrPut(classifier) {
classifier.allFeatures().associateBy { it.name!! }
}[featureName]
}

init {
MetamodelRegistry.registerMapping(IssueNode::class, StarLasuLWLanguage.Issue)
MetamodelRegistry.registerMapping(ParsingResultNode::class, StarLasuLWLanguage.ParsingResult)
Expand Down Expand Up @@ -129,17 +140,6 @@ class LionWebModelConverter(
}
}

companion object {
private val kFeaturesCache = mutableMapOf<Class<*>, Map<String, Feature>>()
private val lwFeaturesCache = mutableMapOf<Classifier<*>, Map<String, LWFeature<*>>>()

fun lwFeatureByName(classifier: Classifier<*>, featureName: String): LWFeature<*>? {
return lwFeaturesCache.getOrPut(classifier) {
classifier.allFeatures().associateBy { it.name!! }
}[featureName]
}
}

fun exportModelToLionWeb(
kolasuTree: KNode,
nodeIdProvider: NodeIdProvider = this.nodeIdProvider,
Expand Down Expand Up @@ -746,11 +746,13 @@ class LionWebModelConverter(
}
ParsingResult::class -> {
val root = data.getOnlyChildByContainmentName(ParsingResult<*>::root.name)
ParsingResult(
val tokens = data.getPropertyValue(data.classifier.getPropertyByName("tokens")!!) as TokensList?
ParsingResultWithTokens(
data.getChildrenByContainmentName(ParsingResult<*>::issues.name).map {
importModelFromLionWeb(it) as Issue
},
if (root != null) importModelFromLionWeb(root) as KNode else null
if (root != null) importModelFromLionWeb(root) as KNode else null,
tokens?.tokens ?: listOf()
)
}
else -> {
Expand All @@ -769,26 +771,41 @@ class LionWebModelConverter(

fun exportIssueToLionweb(issue: Issue): IssueNode {
val issueNode = IssueNode()
issueNode.setPropertyValue(StarLasuLWLanguage.Issue.getPropertyByName("message")!!, issue.message)
issueNode.setPropertyValue(StarLasuLWLanguage.Issue.getPropertyByName("position")!!, issue.position)
setEnumProperty(issueNode, StarLasuLWLanguage.Issue.getPropertyByName("severity")!!, issue.severity)
setEnumProperty(issueNode, StarLasuLWLanguage.Issue.getPropertyByName("type")!!, issue.type)
issueNode.setPropertyValue(StarLasuLWLanguage.Issue.getPropertyByName(Issue::message.name)!!, issue.message)
issueNode.setPropertyValue(StarLasuLWLanguage.Issue.getPropertyByName(Issue::position.name)!!, issue.position)
setEnumProperty(issueNode, StarLasuLWLanguage.Issue.getPropertyByName(Issue::severity.name)!!, issue.severity)
setEnumProperty(issueNode, StarLasuLWLanguage.Issue.getPropertyByName(Issue::type.name)!!, issue.type)
return issueNode
}

fun exportParsingResultToLionweb(pr: ParsingResult<*>): ParsingResultNode {
fun exportParsingResultToLionweb(pr: ParsingResult<*>, tokens: List<KolasuToken> = listOf()): ParsingResultNode {
val resultNode = ParsingResultNode(pr.source)
resultNode.setPropertyValue(StarLasuLWLanguage.ParsingResult.getPropertyByName("code")!!, pr.code)
resultNode.setPropertyValue(
StarLasuLWLanguage.ParsingResult.getPropertyByName(ParsingResult<*>::code.name)!!,
pr.code
)
val root = if (pr.root != null) exportModelToLionWeb(pr.root!!, considerParent = false) else null
resultNode.addChild(StarLasuLWLanguage.ParsingResult.getContainmentByName("root")!!, root)
val issuesContainment = StarLasuLWLanguage.ParsingResult.getContainmentByName("issues")!!
resultNode.addChild(StarLasuLWLanguage.ParsingResult.getContainmentByName(ParsingResult<*>::root.name)!!, root)
val issuesContainment = StarLasuLWLanguage.ParsingResult.getContainmentByName(ParsingResult<*>::issues.name)!!
pr.issues.forEach {
resultNode.addChild(issuesContainment, exportIssueToLionweb(it))
}
resultNode.setPropertyValue(StarLasuLWLanguage.ParsingResult.getPropertyByName("tokens")!!, TokensList(tokens))
return resultNode
}
}

class ParsingResultWithTokens<RootNode : KNode>(
issues: List<Issue>,
root: RootNode?,
val tokens: List<KolasuToken>,
code: String? = null,
incompleteNode: com.strumenta.kolasu.model.Node? = null,
firstStage: FirstStageParsingResult<*>? = null,
time: Long? = null,
source: Source? = null
) : ParsingResult<RootNode>(issues, root, code, incompleteNode, firstStage, time, source)

class IssueNode : BaseNode() {
var type: EnumerationValue? by property("type")
var message: String? by property("message")
Expand Down
34 changes: 34 additions & 0 deletions lionweb/src/main/kotlin/com/strumenta/kolasu/lionweb/StarLasuLWLanguage.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package com.strumenta.kolasu.lionweb
import com.strumenta.kolasu.model.Multiplicity
import com.strumenta.kolasu.model.Point
import com.strumenta.kolasu.model.Position
import com.strumenta.kolasu.parsing.KolasuToken
import com.strumenta.kolasu.parsing.TokenCategory
import com.strumenta.kolasu.validation.IssueSeverity
import com.strumenta.kolasu.validation.IssueType
import io.lionweb.lioncore.java.language.Annotation
Expand All @@ -17,6 +19,7 @@ import io.lionweb.lioncore.java.self.LionCore
import io.lionweb.lioncore.java.serialization.PrimitiveValuesSerialization.PrimitiveDeserializer
import io.lionweb.lioncore.java.serialization.PrimitiveValuesSerialization.PrimitiveSerializer
import io.lionweb.lioncore.kotlin.MetamodelRegistry
import io.lionweb.lioncore.kotlin.addPrimitiveType

private const val PLACEHOLDER_NODE = "PlaceholderNode"

Expand Down Expand Up @@ -63,10 +66,12 @@ object StarLasuLWLanguage : Language("com.strumenta.StarLasu") {
addProperty("position", position, Multiplicity.OPTIONAL)
}

addPrimitiveType(TokensList::class)
ParsingResult = addConcept("ParsingResult").apply {
addContainment("issues", Issue, Multiplicity.MANY)
addContainment("root", ASTNode, Multiplicity.OPTIONAL)
addProperty("code", LionCoreBuiltins.getString(), Multiplicity.OPTIONAL)
addProperty("tokens", MetamodelRegistry.getPrimitiveType(TokensList::class)!!, Multiplicity.OPTIONAL)
}
registerSerializersAndDeserializersInMetamodelRegistry()
}
Expand Down Expand Up @@ -184,4 +189,33 @@ private fun registerSerializersAndDeserializersInMetamodelRegistry() {
positionSerializer,
positionDeserializer
)

val tokensListPrimitiveSerializer = PrimitiveSerializer<TokensList?> { value: TokensList? ->
value?.tokens?.joinToString(";") { kt ->
kt.category.type + "$" + positionSerializer.serialize(kt.position)
}
}
val tokensListPrimitiveDeserializer = PrimitiveDeserializer<TokensList?> { serialized ->
if (serialized == null) {
null
} else {
val tokens = if (serialized.isEmpty()) {
mutableListOf()
} else {
serialized.split(";").map {
val parts = it.split("$")
require(parts.size == 2)
val category = parts[0]
val position = positionDeserializer.deserialize(parts[1])
KolasuToken(TokenCategory(category), position)
}.toMutableList()
}
TokensList(tokens)
}
}
val tlpt = MetamodelRegistry.getPrimitiveType(TokensList::class)
?: throw IllegalStateException("Unknown primitive type class ${TokensList::class}")
MetamodelRegistry.addSerializerAndDeserializer(tlpt, tokensListPrimitiveSerializer, tokensListPrimitiveDeserializer)
}

class TokensList(val tokens: List<KolasuToken>)
26 changes: 24 additions & 2 deletions lionweb/src/test/kotlin/com/strumenta/kolasu/lionweb/LionWebModelConverterTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ import com.strumenta.kolasu.model.SyntheticSource
import com.strumenta.kolasu.model.assignParents
import com.strumenta.kolasu.model.pos
import com.strumenta.kolasu.model.withPosition
import com.strumenta.kolasu.parsing.KolasuToken
import com.strumenta.kolasu.parsing.ParsingResult
import com.strumenta.kolasu.parsing.TokenCategory
import com.strumenta.kolasu.testing.assertASTsAreEqual
import com.strumenta.kolasu.transformation.MissingASTTransformation
import com.strumenta.kolasu.validation.Issue
Expand Down Expand Up @@ -946,11 +948,31 @@ class LionWebModelConverterTest {
}
val converter = LionWebModelConverter()
converter.exportLanguageToLionWeb(kLanguage)
val exported = converter.exportParsingResultToLionweb(parsingResult)
val reimported = converter.importModelFromLionWeb(exported) as ParsingResult<*>
val exported = converter.exportParsingResultToLionweb(
parsingResult,
listOf(
KolasuToken(TokenCategory.STRING_LITERAL, pos(1, 2, 3, 4)),
KolasuToken(TokenCategory.PLAIN_TEXT, pos(3, 5, 6, 7))
)
)
val reimported = converter.importModelFromLionWeb(exported) as ParsingResultWithTokens<*>
assertASTsAreEqual(parsingResult.root!!, reimported.root!!)
assertEquals(3, parsingResult.issues.size)
assertEquals("bla bla", parsingResult.code)
assertEquals(
listOf(
TokenCategory.STRING_LITERAL.type,
TokenCategory.PLAIN_TEXT.type
),
reimported.tokens.map { it.category.type }
)
assertEquals(
listOf(
pos(1, 2, 3, 4),
pos(3, 5, 6, 7)
),
reimported.tokens.map { it.position }
)
}
}

Expand Down
6 changes: 3 additions & 3 deletions semantics/src/test/kotlin/com/strumenta/kolasu/semantics/SymbolResolutionWithSRITest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ class SymbolResolutionWithSRITest {

assertEquals(false, todo2.prerequisite!!.resolved)
symbolResolver.resolve(todoProject, entireTree = true)
assertEquals(true, todo2.prerequisite!!.resolved)
assertEquals(todo1, todo2.prerequisite!!.referred)
assertEquals(true, todo2.prerequisite.resolved)
assertEquals(todo1, todo2.prerequisite.referred)
}

@Test
Expand Down Expand Up @@ -147,7 +147,7 @@ class SymbolResolutionWithSRITest {

assertEquals(false, todo4.prerequisite!!.resolved)
symbolResolver.resolve(todoProjectErrands, entireTree = true)
assertEquals(true, todo4.prerequisite!!.resolved)
assertEquals(true, todo4.prerequisite.resolved)
assertEquals("synthetic_Personal-Source_todos_1", todo4.prerequisite.identifier)
}
}
Expand Down

0 comments on commit c43cae3

Please sign in to comment.