diff --git a/core/src/main/kotlin/com/strumenta/kolasu/parsing/KolasuParser.kt b/core/src/main/kotlin/com/strumenta/kolasu/parsing/KolasuParser.kt index 44b5b239b..7b050a696 100644 --- a/core/src/main/kotlin/com/strumenta/kolasu/parsing/KolasuParser.kt +++ b/core/src/main/kotlin/com/strumenta/kolasu/parsing/KolasuParser.kt @@ -34,10 +34,12 @@ interface TokenFactory { fun categoryOf(t: Token): TokenCategory = TokenCategory.PLAIN_TEXT fun convertToken(t: Token): T +} - private fun convertToken(terminalNode: TerminalNode): T = convertToken(terminalNode.symbol) +open class ANTLRTokenFactory : TokenFactory { + override fun convertToken(t: Token): KolasuANTLRToken = KolasuANTLRToken(categoryOf(t), t) - fun extractTokens(result: ParsingResult<*>): LexingResult? { + fun extractTokens(result: ParsingResult<*>): LexingResult? { val antlrTerminals = mutableListOf() fun extractTokensFromParseTree(pt: ParseTree?) { if (pt is TerminalNode) { @@ -49,20 +51,22 @@ interface TokenFactory { } } - val ptRoot = result.firstStage?.root + val ptRoot = result.firstStage?.root ?: if (result.root?.origin is ParseTreeOrigin) { + (result.root.origin as ParseTreeOrigin).parseTree + } else { + null + } return if (ptRoot != null) { extractTokensFromParseTree(ptRoot) antlrTerminals.sortBy { it.symbol.tokenIndex } val tokens = antlrTerminals.map { convertToken(it) }.toMutableList() - LexingResult(result.issues, tokens, result.code, result.firstStage.lexingTime, result.source) + LexingResult(result.issues, tokens, result.code, result.firstStage?.lexingTime, result.source) } else { null } } -} -open class ANTLRTokenFactory : TokenFactory { - override fun convertToken(t: Token): KolasuANTLRToken = KolasuANTLRToken(categoryOf(t), t) + private fun convertToken(terminalNode: TerminalNode): KolasuANTLRToken = convertToken(terminalNode.symbol) } abstract class KolasuANTLRLexer(val tokenFactory: TokenFactory) : KolasuLexer { diff --git a/core/src/main/kotlin/com/strumenta/kolasu/parsing/Parsing.kt b/core/src/main/kotlin/com/strumenta/kolasu/parsing/Parsing.kt index 23bda56c9..f3bc7f0a2 100644 --- a/core/src/main/kotlin/com/strumenta/kolasu/parsing/Parsing.kt +++ b/core/src/main/kotlin/com/strumenta/kolasu/parsing/Parsing.kt @@ -77,7 +77,7 @@ data class TokenCategory(val type: String) { open class KolasuToken( open val category: TokenCategory, open val position: Position, - open val text: String + open val text: String? = null ) : Serializable /** @@ -155,7 +155,7 @@ class FirstStageParsingResult( * @param firstStage the result of the first parsing stage (from source code to parse tree). * @param time the time spent in the entire parsing process. */ -class ParsingResult( +open class ParsingResult( issues: List, val root: RootNode?, code: String? = null, diff --git a/core/src/test/kotlin/com/strumenta/kolasu/parsing/KolasuParserTest.kt b/core/src/test/kotlin/com/strumenta/kolasu/parsing/KolasuParserTest.kt index 4b03207aa..102862066 100644 --- a/core/src/test/kotlin/com/strumenta/kolasu/parsing/KolasuParserTest.kt +++ b/core/src/test/kotlin/com/strumenta/kolasu/parsing/KolasuParserTest.kt @@ -51,7 +51,7 @@ class KolasuParserTest { """.trimMargin() ) assertNotNull(result) - val lexingResult = parser.tokenFactory.extractTokens(result) + val lexingResult = (parser.tokenFactory as ANTLRTokenFactory).extractTokens(result) assertNotNull(lexingResult) assertEquals(11, lexingResult.tokens.size) val text = lexingResult.tokens.map { it.text } diff --git a/lionweb/src/main/kotlin/com/strumenta/kolasu/lionweb/LionWebModelConverter.kt b/lionweb/src/main/kotlin/com/strumenta/kolasu/lionweb/LionWebModelConverter.kt index b1f8d33c1..306e3434f 100644 --- a/lionweb/src/main/kotlin/com/strumenta/kolasu/lionweb/LionWebModelConverter.kt +++ b/lionweb/src/main/kotlin/com/strumenta/kolasu/lionweb/LionWebModelConverter.kt @@ -19,6 +19,8 @@ import com.strumenta.kolasu.model.isAttribute import com.strumenta.kolasu.model.isContainment import com.strumenta.kolasu.model.isReference import com.strumenta.kolasu.model.nodeOriginalProperties +import com.strumenta.kolasu.parsing.FirstStageParsingResult +import com.strumenta.kolasu.parsing.KolasuToken import com.strumenta.kolasu.parsing.ParsingResult import com.strumenta.kolasu.transformation.MissingASTTransformation import com.strumenta.kolasu.traversing.walk @@ -83,6 +85,15 @@ class LionWebModelConverter( initialLanguageConverter: LionWebLanguageConverter = LionWebLanguageConverter() ) { companion object { + private val kFeaturesCache = mutableMapOf, Map>() + private val lwFeaturesCache = mutableMapOf, Map>>() + + fun lwFeatureByName(classifier: Classifier<*>, featureName: String): LWFeature<*>? { + return lwFeaturesCache.getOrPut(classifier) { + classifier.allFeatures().associateBy { it.name!! } + }[featureName] + } + init { MetamodelRegistry.registerMapping(IssueNode::class, StarLasuLWLanguage.Issue) MetamodelRegistry.registerMapping(ParsingResultNode::class, StarLasuLWLanguage.ParsingResult) @@ -129,17 +140,6 @@ class LionWebModelConverter( } } - companion object { - private val kFeaturesCache = mutableMapOf, Map>() - private val lwFeaturesCache = mutableMapOf, Map>>() - - fun lwFeatureByName(classifier: Classifier<*>, featureName: String): LWFeature<*>? { - return lwFeaturesCache.getOrPut(classifier) { - classifier.allFeatures().associateBy { it.name!! } - }[featureName] - } - } - fun exportModelToLionWeb( kolasuTree: KNode, nodeIdProvider: NodeIdProvider = this.nodeIdProvider, @@ -746,11 +746,13 @@ class LionWebModelConverter( } ParsingResult::class -> { val root = data.getOnlyChildByContainmentName(ParsingResult<*>::root.name) - ParsingResult( + val tokens = data.getPropertyValue(data.classifier.getPropertyByName("tokens")!!) as TokensList? + ParsingResultWithTokens( data.getChildrenByContainmentName(ParsingResult<*>::issues.name).map { importModelFromLionWeb(it) as Issue }, - if (root != null) importModelFromLionWeb(root) as KNode else null + if (root != null) importModelFromLionWeb(root) as KNode else null, + tokens?.tokens ?: listOf() ) } else -> { @@ -769,26 +771,41 @@ class LionWebModelConverter( fun exportIssueToLionweb(issue: Issue): IssueNode { val issueNode = IssueNode() - issueNode.setPropertyValue(StarLasuLWLanguage.Issue.getPropertyByName("message")!!, issue.message) - issueNode.setPropertyValue(StarLasuLWLanguage.Issue.getPropertyByName("position")!!, issue.position) - setEnumProperty(issueNode, StarLasuLWLanguage.Issue.getPropertyByName("severity")!!, issue.severity) - setEnumProperty(issueNode, StarLasuLWLanguage.Issue.getPropertyByName("type")!!, issue.type) + issueNode.setPropertyValue(StarLasuLWLanguage.Issue.getPropertyByName(Issue::message.name)!!, issue.message) + issueNode.setPropertyValue(StarLasuLWLanguage.Issue.getPropertyByName(Issue::position.name)!!, issue.position) + setEnumProperty(issueNode, StarLasuLWLanguage.Issue.getPropertyByName(Issue::severity.name)!!, issue.severity) + setEnumProperty(issueNode, StarLasuLWLanguage.Issue.getPropertyByName(Issue::type.name)!!, issue.type) return issueNode } - fun exportParsingResultToLionweb(pr: ParsingResult<*>): ParsingResultNode { + fun exportParsingResultToLionweb(pr: ParsingResult<*>, tokens: List = listOf()): ParsingResultNode { val resultNode = ParsingResultNode(pr.source) - resultNode.setPropertyValue(StarLasuLWLanguage.ParsingResult.getPropertyByName("code")!!, pr.code) + resultNode.setPropertyValue( + StarLasuLWLanguage.ParsingResult.getPropertyByName(ParsingResult<*>::code.name)!!, + pr.code + ) val root = if (pr.root != null) exportModelToLionWeb(pr.root!!, considerParent = false) else null - resultNode.addChild(StarLasuLWLanguage.ParsingResult.getContainmentByName("root")!!, root) - val issuesContainment = StarLasuLWLanguage.ParsingResult.getContainmentByName("issues")!! + resultNode.addChild(StarLasuLWLanguage.ParsingResult.getContainmentByName(ParsingResult<*>::root.name)!!, root) + val issuesContainment = StarLasuLWLanguage.ParsingResult.getContainmentByName(ParsingResult<*>::issues.name)!! pr.issues.forEach { resultNode.addChild(issuesContainment, exportIssueToLionweb(it)) } + resultNode.setPropertyValue(StarLasuLWLanguage.ParsingResult.getPropertyByName("tokens")!!, TokensList(tokens)) return resultNode } } +class ParsingResultWithTokens( + issues: List, + root: RootNode?, + val tokens: List, + code: String? = null, + incompleteNode: com.strumenta.kolasu.model.Node? = null, + firstStage: FirstStageParsingResult<*>? = null, + time: Long? = null, + source: Source? = null +) : ParsingResult(issues, root, code, incompleteNode, firstStage, time, source) + class IssueNode : BaseNode() { var type: EnumerationValue? by property("type") var message: String? by property("message") diff --git a/lionweb/src/main/kotlin/com/strumenta/kolasu/lionweb/StarLasuLWLanguage.kt b/lionweb/src/main/kotlin/com/strumenta/kolasu/lionweb/StarLasuLWLanguage.kt index 34ce5caee..51bdd143b 100644 --- a/lionweb/src/main/kotlin/com/strumenta/kolasu/lionweb/StarLasuLWLanguage.kt +++ b/lionweb/src/main/kotlin/com/strumenta/kolasu/lionweb/StarLasuLWLanguage.kt @@ -3,6 +3,8 @@ package com.strumenta.kolasu.lionweb import com.strumenta.kolasu.model.Multiplicity import com.strumenta.kolasu.model.Point import com.strumenta.kolasu.model.Position +import com.strumenta.kolasu.parsing.KolasuToken +import com.strumenta.kolasu.parsing.TokenCategory import com.strumenta.kolasu.validation.IssueSeverity import com.strumenta.kolasu.validation.IssueType import io.lionweb.lioncore.java.language.Annotation @@ -17,6 +19,7 @@ import io.lionweb.lioncore.java.self.LionCore import io.lionweb.lioncore.java.serialization.PrimitiveValuesSerialization.PrimitiveDeserializer import io.lionweb.lioncore.java.serialization.PrimitiveValuesSerialization.PrimitiveSerializer import io.lionweb.lioncore.kotlin.MetamodelRegistry +import io.lionweb.lioncore.kotlin.addPrimitiveType private const val PLACEHOLDER_NODE = "PlaceholderNode" @@ -63,10 +66,12 @@ object StarLasuLWLanguage : Language("com.strumenta.StarLasu") { addProperty("position", position, Multiplicity.OPTIONAL) } + addPrimitiveType(TokensList::class) ParsingResult = addConcept("ParsingResult").apply { addContainment("issues", Issue, Multiplicity.MANY) addContainment("root", ASTNode, Multiplicity.OPTIONAL) addProperty("code", LionCoreBuiltins.getString(), Multiplicity.OPTIONAL) + addProperty("tokens", MetamodelRegistry.getPrimitiveType(TokensList::class)!!, Multiplicity.OPTIONAL) } registerSerializersAndDeserializersInMetamodelRegistry() } @@ -184,4 +189,33 @@ private fun registerSerializersAndDeserializersInMetamodelRegistry() { positionSerializer, positionDeserializer ) + + val tokensListPrimitiveSerializer = PrimitiveSerializer { value: TokensList? -> + value?.tokens?.joinToString(";") { kt -> + kt.category.type + "$" + positionSerializer.serialize(kt.position) + } + } + val tokensListPrimitiveDeserializer = PrimitiveDeserializer { serialized -> + if (serialized == null) { + null + } else { + val tokens = if (serialized.isEmpty()) { + mutableListOf() + } else { + serialized.split(";").map { + val parts = it.split("$") + require(parts.size == 2) + val category = parts[0] + val position = positionDeserializer.deserialize(parts[1]) + KolasuToken(TokenCategory(category), position) + }.toMutableList() + } + TokensList(tokens) + } + } + val tlpt = MetamodelRegistry.getPrimitiveType(TokensList::class) + ?: throw IllegalStateException("Unknown primitive type class ${TokensList::class}") + MetamodelRegistry.addSerializerAndDeserializer(tlpt, tokensListPrimitiveSerializer, tokensListPrimitiveDeserializer) } + +class TokensList(val tokens: List) diff --git a/lionweb/src/test/kotlin/com/strumenta/kolasu/lionweb/LionWebModelConverterTest.kt b/lionweb/src/test/kotlin/com/strumenta/kolasu/lionweb/LionWebModelConverterTest.kt index d8f605b80..1633b4cb1 100644 --- a/lionweb/src/test/kotlin/com/strumenta/kolasu/lionweb/LionWebModelConverterTest.kt +++ b/lionweb/src/test/kotlin/com/strumenta/kolasu/lionweb/LionWebModelConverterTest.kt @@ -13,7 +13,9 @@ import com.strumenta.kolasu.model.SyntheticSource import com.strumenta.kolasu.model.assignParents import com.strumenta.kolasu.model.pos import com.strumenta.kolasu.model.withPosition +import com.strumenta.kolasu.parsing.KolasuToken import com.strumenta.kolasu.parsing.ParsingResult +import com.strumenta.kolasu.parsing.TokenCategory import com.strumenta.kolasu.testing.assertASTsAreEqual import com.strumenta.kolasu.transformation.MissingASTTransformation import com.strumenta.kolasu.validation.Issue @@ -946,11 +948,31 @@ class LionWebModelConverterTest { } val converter = LionWebModelConverter() converter.exportLanguageToLionWeb(kLanguage) - val exported = converter.exportParsingResultToLionweb(parsingResult) - val reimported = converter.importModelFromLionWeb(exported) as ParsingResult<*> + val exported = converter.exportParsingResultToLionweb( + parsingResult, + listOf( + KolasuToken(TokenCategory.STRING_LITERAL, pos(1, 2, 3, 4)), + KolasuToken(TokenCategory.PLAIN_TEXT, pos(3, 5, 6, 7)) + ) + ) + val reimported = converter.importModelFromLionWeb(exported) as ParsingResultWithTokens<*> assertASTsAreEqual(parsingResult.root!!, reimported.root!!) assertEquals(3, parsingResult.issues.size) assertEquals("bla bla", parsingResult.code) + assertEquals( + listOf( + TokenCategory.STRING_LITERAL.type, + TokenCategory.PLAIN_TEXT.type + ), + reimported.tokens.map { it.category.type } + ) + assertEquals( + listOf( + pos(1, 2, 3, 4), + pos(3, 5, 6, 7) + ), + reimported.tokens.map { it.position } + ) } } diff --git a/semantics/src/test/kotlin/com/strumenta/kolasu/semantics/SymbolResolutionWithSRITest.kt b/semantics/src/test/kotlin/com/strumenta/kolasu/semantics/SymbolResolutionWithSRITest.kt index 73dbd36d1..e72dd581f 100644 --- a/semantics/src/test/kotlin/com/strumenta/kolasu/semantics/SymbolResolutionWithSRITest.kt +++ b/semantics/src/test/kotlin/com/strumenta/kolasu/semantics/SymbolResolutionWithSRITest.kt @@ -53,8 +53,8 @@ class SymbolResolutionWithSRITest { assertEquals(false, todo2.prerequisite!!.resolved) symbolResolver.resolve(todoProject, entireTree = true) - assertEquals(true, todo2.prerequisite!!.resolved) - assertEquals(todo1, todo2.prerequisite!!.referred) + assertEquals(true, todo2.prerequisite.resolved) + assertEquals(todo1, todo2.prerequisite.referred) } @Test @@ -147,7 +147,7 @@ class SymbolResolutionWithSRITest { assertEquals(false, todo4.prerequisite!!.resolved) symbolResolver.resolve(todoProjectErrands, entireTree = true) - assertEquals(true, todo4.prerequisite!!.resolved) + assertEquals(true, todo4.prerequisite.resolved) assertEquals("synthetic_Personal-Source_todos_1", todo4.prerequisite.identifier) } }