Skip to content

Commit

Permalink
feat(fossid-webapp): List snippets from FossID lazily
Browse files Browse the repository at this point in the history 
Currently, the FossID snippets are all listed in a first step, including
their matching lines. Then in a second step, they are mapped to ORT
snippets.
A future commit is going to introduce a limit for listing snippets. If this
limit is applied to the current logic (i.e. when listing the snippets), it
is going to clash with the snippet choice feature: chosen snippets are
removed from the results and thus should not be counted when enforcing the
limit.
Therefore, this commit makes the listed snippets a Flow. When mapping the
snippets and applying the snippet choices, the current snippet count can
then be compared to the limit and additional snippets can lazily be
fetched.

Signed-off-by: Nicolas Nobelis <nicolas.nobelis@bosch.com>
  • Loading branch information
@nnobelis @oheger-bosch
nnobelis authored and oheger-bosch committed May 15, 2024
1 parent 10cef09 commit 6a53cc0
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 54 deletions.
52 changes: 27 additions & 25 deletions plugins/scanners/fossid/src/main/kotlin/FossId.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeoutOrNull

Expand Down Expand Up @@ -857,36 +858,37 @@ class FossId internal constructor(
}

val matchedLines = mutableMapOf<Int, MatchedLines>()
val snippets = runBlocking(Dispatchers.IO) {
pendingFiles.map {
async {
logger.info { "Listing snippet for $it..." }
val snippetResponse = service.listSnippets(config.user, config.apiKey, scanCode, it)
.checkResponse("list snippets")
val snippets = checkNotNull(snippetResponse.data) {
"Snippet could not be listed. Response was ${snippetResponse.message}."
}
logger.info { "${snippets.size} snippets." }
val pendingFilesIterator = pendingFiles.iterator()
val snippets = flow {
while (pendingFilesIterator.hasNext()) {
val file = pendingFilesIterator.next()
logger.info { "Listing snippet for $file..." }

val snippetResponse = service.listSnippets(config.user, config.apiKey, scanCode, file)
.checkResponse("list snippets")
val snippets = checkNotNull(snippetResponse.data) {
"Snippet could not be listed. Response was ${snippetResponse.message}."
}
logger.info { "${snippets.size} snippets." }

val filteredSnippets = snippets.filterTo(mutableSetOf()) { it.matchType.isValidType() }
val filteredSnippets = snippets.filterTo(mutableSetOf()) { it.matchType.isValidType() }

if (config.fetchSnippetMatchedLines) {
logger.info { "Listing snippet matched lines for $it..." }
if (config.fetchSnippetMatchedLines) {
logger.info { "Listing snippet matched lines for $file..." }

filteredSnippets.filter { it.matchType == MatchType.PARTIAL }.map { snippet ->
val matchedLinesResponse =
service.listMatchedLines(config.user, config.apiKey, scanCode, it, snippet.id)
.checkResponse("list snippets matched lines")
val lines = checkNotNull(matchedLinesResponse.data) {
"Matched lines could not be listed. Response was ${matchedLinesResponse.message}."
}
matchedLines[snippet.id] = lines
filteredSnippets.filter { it.matchType == MatchType.PARTIAL }.map { snippet ->
val matchedLinesResponse =
service.listMatchedLines(config.user, config.apiKey, scanCode, file, snippet.id)
.checkResponse("list snippets matched lines")
val lines = checkNotNull(matchedLinesResponse.data) {
"Matched lines could not be listed. Response was ${matchedLinesResponse.message}."
}
matchedLines[snippet.id] = lines
}

it to filteredSnippets
}
}.awaitAll().toMap()

emit(file to filteredSnippets.toSet())
}
}

return RawResults(
Expand All @@ -903,7 +905,7 @@ class FossId internal constructor(
* Construct the [ScanSummary] for this FossID scan.
*/
@Suppress("LongParameterList")
private fun createResultSummary(
private suspend fun createResultSummary(
startTime: Instant,
provenance: Provenance,
rawResults: RawResults,
Expand Down
16 changes: 12 additions & 4 deletions plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ package org.ossreviewtoolkit.plugins.scanners.fossid

import java.lang.invoke.MethodHandles

import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.map

import org.apache.logging.log4j.kotlin.loggerOf

import org.ossreviewtoolkit.clients.fossid.model.identification.identifiedFiles.IdentifiedFile
Expand Down Expand Up @@ -64,7 +67,7 @@ internal data class RawResults(
val markedAsIdentifiedFiles: List<MarkedAsIdentifiedFile>,
val listIgnoredFiles: List<IgnoredFile>,
val listPendingFiles: List<String>,
val listSnippets: Map<String, Set<Snippet>>,
val listSnippets: Flow<Pair<String, Set<Snippet>>>,
val snippetMatchedLines: Map<Int, MatchedLines> = emptyMap()
)

Expand Down Expand Up @@ -162,16 +165,17 @@ private fun mapLicense(
* Map the raw snippets to ORT [SnippetFinding]s. If a snippet license cannot be parsed, an issues is added to [issues].
* [LicenseFinding]s due to chosen snippets will be added to [snippetLicenseFindings].
*/
internal fun mapSnippetFindings(
internal suspend fun mapSnippetFindings(
rawResults: RawResults,
issues: MutableList<Issue>,
detectedLicenseMapping: Map<String, String>,
snippetChoices: List<SnippetChoice>,
snippetLicenseFindings: MutableSet<LicenseFinding>
): Set<SnippetFinding> {
val remainingSnippetChoices = snippetChoices.toMutableList()
val allFindings = mutableSetOf<SnippetFinding>()

return rawResults.listSnippets.flatMap { (file, rawSnippets) ->
rawResults.listSnippets.map { (file, rawSnippets) ->
val findings = mutableMapOf<TextLocation, MutableSet<OrtSnippet>>()

rawSnippets.forEach { snippet ->
Expand Down Expand Up @@ -279,7 +283,11 @@ internal fun mapSnippetFindings(
}

findings.map { SnippetFinding(it.key, it.value) }
}.toSet().also {
}.collect {
allFindings += it
}

return allFindings.also {
remainingSnippetChoices.forEach { snippetChoice ->
// The issue is created only if the chosen snippet does not correspond to a file marked by a previous run.
val isNotOldMarkedAsIdentifiedFile = rawResults.markedAsIdentifiedFiles.none { markedFile ->
Expand Down
4 changes: 3 additions & 1 deletion plugins/scanners/fossid/src/test/kotlin/FossIdLicenseMappingTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import io.kotest.matchers.should
import io.kotest.matchers.shouldBe
import io.kotest.matchers.string.shouldStartWith

import kotlinx.coroutines.flow.flowOf

import org.ossreviewtoolkit.clients.fossid.model.result.LicenseCategory
import org.ossreviewtoolkit.clients.fossid.model.result.MatchType
import org.ossreviewtoolkit.clients.fossid.model.result.Snippet
Expand Down Expand Up @@ -150,5 +152,5 @@ private fun createSnippet(license: String): RawResults {
null,
null
)
return RawResults(emptyList(), emptyList(), emptyList(), emptyList(), mapOf(FILE_PATH to setOf(snippet)))
return RawResults(emptyList(), emptyList(), emptyList(), emptyList(), flowOf(FILE_PATH to setOf(snippet)))
}
52 changes: 28 additions & 24 deletions plugins/scanners/fossid/src/test/kotlin/FossIdSnippetMappingTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import io.kotest.matchers.collections.shouldHaveSize
import io.kotest.matchers.should
import io.kotest.matchers.shouldBe

import kotlinx.coroutines.flow.flowOf

import org.ossreviewtoolkit.clients.fossid.PolymorphicList
import org.ossreviewtoolkit.clients.fossid.model.result.MatchType
import org.ossreviewtoolkit.clients.fossid.model.result.MatchedLines
Expand All @@ -38,32 +40,34 @@ class FossIdSnippetMappingTest : WordSpec({
"mapSnippetFindings" should {
"group snippets by source file location" {
val issues = mutableListOf<Issue>()
val listSnippets = mapOf(
"src/main/java/Tokenizer.java" to setOf(
createSnippet(
1,
MatchType.FULL,
"pkg:github/vdurmont/semver4j@3.1.0",
"MIT",
"src/main/java/com/vdurmont/semver4j/Tokenizer.java"
val listSnippets = flowOf(
"src/main/java/Tokenizer.java" to
setOf(
createSnippet(
1,
MatchType.FULL,
"pkg:github/vdurmont/semver4j@3.1.0",
"MIT",
"src/main/java/com/vdurmont/semver4j/Tokenizer.java"
),
createSnippet(
2,
MatchType.FULL,
"pkg:maven/com.vdurmont/semver4j@3.1.0",
"MIT",
"com/vdurmont/semver4j/Tokenizer.java"
)
),
createSnippet(
2,
MatchType.FULL,
"pkg:maven/com.vdurmont/semver4j@3.1.0",
"MIT",
"com/vdurmont/semver4j/Tokenizer.java"
)
),
"src/main/java/com/vdurmont/semver4j/Requirement.java" to setOf(
createSnippet(
3,
MatchType.PARTIAL,
"pkg:github/vdurmont/semver4j@3.1.0",
"MIT",
"com/vdurmont/semver4j/Requirement.java"
"src/main/java/com/vdurmont/semver4j/Requirement.java" to
setOf(
createSnippet(
3,
MatchType.PARTIAL,
"pkg:github/vdurmont/semver4j@3.1.0",
"MIT",
"com/vdurmont/semver4j/Requirement.java"
)
)
)
)
val localFile = ((1..24) + (45..675)).toPolymorphicList()
val remoteFile = (1..655).toPolymorphicList()
Expand Down

0 comments on commit 6a53cc0

Please sign in to comment.