Skip to content

Commit

Permalink
refactor(bundler): Migrate from Jackson to KxS
Browse files Browse the repository at this point in the history
This avoids the use of reflection which increases performance and eases
creation of GraalVM native images. As a bonus, the model is implemented
via type-safe data classes now.

Signed-off-by: Sebastian Schuberth <sebastian@doubleopen.org>
  • Loading branch information
sschuberth committed Jul 2, 2024
1 parent 1699c84 commit 065e1ca
Show file tree
Hide file tree
Showing 5 changed files with 201 additions and 51 deletions.
2 changes: 2 additions & 0 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ jiraRestClient = "5.2.7"
jruby = "9.4.7.0"
jslt = "0.1.14"
jsonSchemaValidator = "1.4.3"
kaml = "0.60.0"
kotest = "5.9.1"
kotlinxCoroutines = "1.8.1"
kotlinxHtml = "0.11.0"
Expand Down Expand Up @@ -131,6 +132,7 @@ kotlinx-html = { module = "org.jetbrains.kotlinx:kotlinx-html-jvm", version.ref
kotlinx-serialization-core = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version.ref = "kotlinxSerialization" }
kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerialization" }
kotlinx-serialization-toml = { module = "net.peanuuutz.tomlkt:tomlkt", version.ref = "tomlkt" }
kotlinx-serialization-yaml = { module = "com.charleskorn.kaml:kaml", version.ref = "kaml" }
ks3-jdk = { module = "io.ks3:ks3-jdk", version.ref = "ks3" }
ks3-standard = { module = "io.ks3:ks3-standard", version.ref = "ks3" }
ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" }
Expand Down
8 changes: 5 additions & 3 deletions plugins/package-managers/bundler/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
plugins {
// Apply precompiled plugins.
id("ort-library-conventions")

// Apply third-party plugins.
alias(libs.plugins.kotlinSerialization)
}

dependencies {
Expand All @@ -31,10 +34,9 @@ dependencies {
implementation(projects.utils.ortUtils)
implementation(projects.utils.spdxUtils)

implementation(libs.jackson.core)
implementation(libs.jackson.databind)
implementation(libs.jackson.dataformat.yaml)
implementation(libs.jruby)
implementation(libs.kotlinx.serialization.core)
implementation(libs.kotlinx.serialization.yaml)

funTestImplementation(testFixtures(projects.analyzer))
}
89 changes: 43 additions & 46 deletions plugins/package-managers/bundler/src/main/kotlin/Bundler.kt
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

package org.ossreviewtoolkit.plugins.packagemanagers.bundler

import com.fasterxml.jackson.databind.JsonNode
import com.charleskorn.kaml.Yaml

import java.io.ByteArrayOutputStream
import java.io.File
Expand All @@ -29,6 +29,8 @@ import java.net.HttpURLConnection

import kotlin.time.measureTime

import kotlinx.serialization.serializer

import org.apache.logging.log4j.kotlin.logger

import org.jruby.embed.LocalContextScope
Expand All @@ -53,12 +55,9 @@ import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
import org.ossreviewtoolkit.model.config.PackageManagerConfiguration
import org.ossreviewtoolkit.model.config.RepositoryConfiguration
import org.ossreviewtoolkit.model.createAndLogIssue
import org.ossreviewtoolkit.model.fromYaml
import org.ossreviewtoolkit.model.orEmpty
import org.ossreviewtoolkit.model.yamlMapper
import org.ossreviewtoolkit.utils.common.AlphaNumericComparator
import org.ossreviewtoolkit.utils.common.collectMessages
import org.ossreviewtoolkit.utils.common.textValueOrEmpty
import org.ossreviewtoolkit.utils.ort.HttpDownloadError
import org.ossreviewtoolkit.utils.ort.OkHttpClientHelper
import org.ossreviewtoolkit.utils.ort.downloadText
Expand All @@ -85,6 +84,9 @@ private const val BUNDLER_GEM_NAME = "bundler"
*/
internal const val BUNDLER_LOCKFILE_NAME = "Gemfile.lock"

// TODO: Remove this again once available upstream.
private inline fun <reified T> Yaml.decodeFromString(string: String): T = decodeFromString(serializer<T>(), string)

private fun runScriptCode(code: String, workingDir: File? = null): String {
val bytes = ByteArrayOutputStream()

Expand Down Expand Up @@ -306,15 +308,16 @@ class Bundler(
}

private fun getDependencyGroups(workingDir: File): Map<String, List<String>> =
runScriptResource(ROOT_DEPENDENCIES_SCRIPT, workingDir).fromYaml()
YAML.decodeFromString(runScriptResource(ROOT_DEPENDENCIES_SCRIPT, workingDir))

private fun resolveGemsInfo(workingDir: File): MutableMap<String, GemInfo> {
val stdout = runScriptResource(RESOLVE_DEPENDENCIES_SCRIPT, workingDir)

// The metadata produced by the "resolve_dependencies.rb" script separates specs for packages with the "\0"
// character as delimiter.
val gemsInfo = stdout.split('\u0000').map {
GemInfo.createFromMetadata(yamlMapper.readTree(it))
val spec = YAML.decodeFromString<GemSpec>(it)
GemInfo.createFromMetadata(spec)
}.associateByTo(mutableMapOf()) {
it.name
}
Expand Down Expand Up @@ -365,7 +368,8 @@ class Bundler(
val url = "https://rubygems.org/api/v2/rubygems/$name/versions/$version.yaml?platform=ruby"

return okHttpClient.downloadText(url).mapCatching {
GemInfo.createFromGem(yamlMapper.readTree(it))
val details = YAML.decodeFromString<VersionDetails>(it)
GemInfo.createFromGem(details)
}.onFailure {
val error = (it as? HttpDownloadError) ?: run {
logger.warn { "Unable to retrieve metadata for gem '$name' from RubyGems: ${it.message}" }
Expand Down Expand Up @@ -416,65 +420,58 @@ internal data class GemInfo(
val artifact: RemoteArtifact
) {
companion object {
fun createFromMetadata(node: JsonNode): GemInfo {
val runtimeDependencies = node["dependencies"]?.asIterable()?.mapNotNull { dependency ->
dependency["name"]?.textValue()?.takeIf { dependency["type"]?.textValue() == ":runtime" }
}?.toSet()
fun createFromMetadata(spec: GemSpec): GemInfo {
val runtimeDependencies = spec.dependencies.mapNotNullTo(mutableSetOf()) { (name, type) ->
name.takeIf { type == VersionDetails.Scope.RUNTIME.toString() }
}

val homepage = spec.homepage.orEmpty()

val homepage = node["homepage"].textValueOrEmpty()
return GemInfo(
node["name"].textValue(),
node["version"]["version"].textValue(),
spec.name,
spec.version.version,
homepage,
node["authors"]?.toList().mapToSetOfNotEmptyStrings(),
node["licenses"]?.toList().mapToSetOfNotEmptyStrings(),
node["description"].textValueOrEmpty(),
runtimeDependencies.orEmpty(),
spec.authors.mapToSetOfNotEmptyStrings(),
spec.licenses.mapToSetOfNotEmptyStrings(),
spec.description.orEmpty(),
runtimeDependencies,
VcsHost.parseUrl(homepage),
RemoteArtifact.EMPTY
)
}

fun createFromGem(node: JsonNode): GemInfo {
val runtimeDependencies = node["dependencies"]?.get("runtime")?.mapNotNull { dependency ->
dependency["name"]?.textValue()
}?.toSet()
fun createFromGem(details: VersionDetails): GemInfo {
val runtimeDependencies = details.dependencies[VersionDetails.Scope.RUNTIME]
?.mapTo(mutableSetOf()) { it.name }
.orEmpty()

val vcs = sequenceOf(node["source_code_uri"], node["homepage_uri"]).mapNotNull { uri ->
uri?.textValue()?.takeIf { it.isNotEmpty() }
}.firstOrNull()?.let {
VcsHost.parseUrl(it)
}.orEmpty()
val vcs = listOfNotNull(details.sourceCodeUri, details.homepageUri)
.mapToSetOfNotEmptyStrings()
.firstOrNull()
?.let { VcsHost.parseUrl(it) }
.orEmpty()

val artifact = if (node.hasNonNull("gem_uri") && node.hasNonNull("sha")) {
val sha = node["sha"].textValue()
RemoteArtifact(node["gem_uri"].textValue(), Hash.create(sha))
val artifact = if (details.gemUri != null && details.sha != null) {
RemoteArtifact(details.gemUri, Hash.create(details.sha))
} else {
RemoteArtifact.EMPTY
}

return GemInfo(
node["name"].textValue(),
node["version"].textValue(),
node["homepage_uri"].textValueOrEmpty(),
node["authors"].textValueOrEmpty().split(',').mapToSetOfNotEmptyStrings(),
node["licenses"]?.toList().mapToSetOfNotEmptyStrings(),
node["info"].textValueOrEmpty(),
runtimeDependencies.orEmpty(),
details.name,
details.version,
details.homepageUri.orEmpty(),
details.authors?.split(',').mapToSetOfNotEmptyStrings(),
details.licenses.mapToSetOfNotEmptyStrings(),
details.info.orEmpty(),
runtimeDependencies,
vcs,
artifact
)
}

private inline fun <reified T> Collection<T>?.mapToSetOfNotEmptyStrings(): Set<String> =
this?.mapNotNullTo(mutableSetOf()) { entry ->
val text = when (T::class) {
JsonNode::class -> (entry as JsonNode).textValue()
else -> entry.toString()
}

text?.trim()?.takeIf { it.isNotEmpty() }
}.orEmpty()
private fun Collection<String>?.mapToSetOfNotEmptyStrings(): Set<String> =
this?.mapNotNullTo(mutableSetOf()) { string -> string.trim().takeUnless { it.isEmpty() } }.orEmpty()
}

fun merge(other: GemInfo): GemInfo {
Expand Down
145 changes: 145 additions & 0 deletions plugins/package-managers/bundler/src/main/kotlin/Model.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
/*
* Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.plugins.packagemanagers.bundler

import com.charleskorn.kaml.Yaml
import com.charleskorn.kaml.YamlConfiguration
import com.charleskorn.kaml.YamlNamingStrategy

import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable

internal val YAML = Yaml(
configuration = YamlConfiguration(strictMode = false, yamlNamingStrategy = YamlNamingStrategy.SnakeCase)
)

/**
* The information for a Gem as typically defined in a ".gemspec" file, see
* - https://guides.rubygems.org/specification-reference/
* - https://github.com/rubygems/rubygems/blob/95128500fba28e3bb03d3bbfcf8536d00a893035/lib/rubygems/specification.rb#L19-L40
*/
@Serializable
internal data class GemSpec(
/** The Gem’s name. */
val name: String,

/** The Gem’s specified version. */
val version: Version,

/** A long description of this Gem. */
val description: String? = null,

/** A short summary of this Gem’s description. */
val summary: String? = null,

/** The URI of this Gem’s homepage. */
val homepage: String? = null,

/** A list of authors for this Gem. */
val authors: List<String> = emptyList(),

/** The license(s) for the Gem. Each license must be a short name, no more than 64 characters. */
val licenses: List<String> = emptyList(),

/** The list of specified dependencies. */
val dependencies: List<Dependency> = emptyList()
) {
@Serializable
data class Version(
/** The version string, containing numbers and periods, such as "1.0.0.pre" for a prerelease. */
val version: String
)

@Serializable
data class Dependency(
/** The name of the dependency. */
val name: String,

/** The type of the dependency as a Ruby symbol, one of ":development" or ":runtime". */
val type: String
)
}

/**
* Version details for a specific Gem version, see
* - https://guides.rubygems.org/rubygems-org-api-v2/
* - https://github.com/rubygems/rubygems.org/blob/1f308c8d55403ccc04df407399bcafce87aa5016/app/models/rubygem.rb#L211-L239
*/
@Serializable
internal data class VersionDetails(
/** The Gem’s name. */
val name: String,

/** The version string, containing numbers and periods, such as "1.0.0.pre" for a prerelease. */
val version: String,

/** A comma-separated list of authors for this Gem. */
val authors: String? = null,

/** A long description of this Gem. */
val description: String? = null,

/** A short summary of this Gem’s description. */
val summary: String? = null,

/** A synthetic field that contains either the descript or summary, if present. */
val info: String? = null,

/** The license(s) for the Gem. Each license must be a short name, no more than 64 characters. */
val licenses: List<String> = emptyList(),

/** The SHA256 hash of the Gem artifact. */
val sha: String? = null,

/** The download URI of the Gem artifact. */
val gemUri: String? = null,

/** The URI of this Gem’s homepage. */
val homepageUri: String? = null,

/** The URI of this Gem’s source code. */
val sourceCodeUri: String? = null,

/** A map of dependencies per scope. */
val dependencies: Map<Scope, List<Dependency>> = emptyMap()
) {
@Serializable
enum class Scope {
/** Dependencies required during development. */
@SerialName("development")
DEVELOPMENT,

/** Dependencies required at runtime. */
@SerialName("runtime")
RUNTIME;

/** A string representation as a Ruby symbol for convenient dependency type matching. */
override fun toString() = ":${name.lowercase()}"
}

@Serializable
data class Dependency(
/** The name of the dependency Gem. */
val name: String,

/** The version requirements string of the dependency Gem. */
val requirements: String? = null
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

package org.ossreviewtoolkit.plugins.packagemanagers.bundler

import com.charleskorn.kaml.decodeFromStream

import io.kotest.core.spec.style.WordSpec
import io.kotest.engine.spec.tempdir
import io.kotest.matchers.shouldBe
Expand All @@ -30,7 +32,6 @@ import org.ossreviewtoolkit.model.HashAlgorithm
import org.ossreviewtoolkit.model.RemoteArtifact
import org.ossreviewtoolkit.model.VcsInfo
import org.ossreviewtoolkit.model.VcsType
import org.ossreviewtoolkit.model.readValue

class BundlerTest : WordSpec({
"parseBundlerVersionFromLockfile()" should {
Expand Down Expand Up @@ -59,8 +60,11 @@ class BundlerTest : WordSpec({
"createFromGem()" should {
"parse YAML metadata for a Gem correctly" {
val rubyGemsFile = File("src/test/assets/rspec-3.7.0.yaml")
val details = rubyGemsFile.inputStream().use {
YAML.decodeFromStream<VersionDetails>(it)
}

val gemInfo = GemInfo.createFromGem(rubyGemsFile.readValue())
val gemInfo = GemInfo.createFromGem(details)

gemInfo shouldBe GemInfo(
name = "rspec",
Expand Down

0 comments on commit 065e1ca

Please sign in to comment.