diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8f7e9ffb4c30..33b934a3a79a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -32,6 +32,7 @@ jiraRestClient = "5.2.7" jruby = "9.4.7.0" jslt = "0.1.14" jsonSchemaValidator = "1.4.3" +kaml = "0.60.0" kotest = "5.9.1" kotlinxCoroutines = "1.8.1" kotlinxHtml = "0.11.0" @@ -131,6 +132,7 @@ kotlinx-html = { module = "org.jetbrains.kotlinx:kotlinx-html-jvm", version.ref kotlinx-serialization-core = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version.ref = "kotlinxSerialization" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerialization" } kotlinx-serialization-toml = { module = "net.peanuuutz.tomlkt:tomlkt", version.ref = "tomlkt" } +kotlinx-serialization-yaml = { module = "com.charleskorn.kaml:kaml", version.ref = "kaml" } ks3-jdk = { module = "io.ks3:ks3-jdk", version.ref = "ks3" } ks3-standard = { module = "io.ks3:ks3-standard", version.ref = "ks3" } ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } diff --git a/plugins/package-managers/bundler/build.gradle.kts b/plugins/package-managers/bundler/build.gradle.kts index b204dbca63d0..f84671eb35c0 100644 --- a/plugins/package-managers/bundler/build.gradle.kts +++ b/plugins/package-managers/bundler/build.gradle.kts @@ -20,6 +20,9 @@ plugins { // Apply precompiled plugins. id("ort-library-conventions") + + // Apply third-party plugins. + alias(libs.plugins.kotlinSerialization) } dependencies { @@ -31,10 +34,9 @@ dependencies { implementation(projects.utils.ortUtils) implementation(projects.utils.spdxUtils) - implementation(libs.jackson.core) - implementation(libs.jackson.databind) - implementation(libs.jackson.dataformat.yaml) implementation(libs.jruby) + implementation(libs.kotlinx.serialization.core) + implementation(libs.kotlinx.serialization.yaml) funTestImplementation(testFixtures(projects.analyzer)) } diff --git a/plugins/package-managers/bundler/src/main/kotlin/Bundler.kt b/plugins/package-managers/bundler/src/main/kotlin/Bundler.kt index 0af713b31144..480d517ac872 100644 --- a/plugins/package-managers/bundler/src/main/kotlin/Bundler.kt +++ b/plugins/package-managers/bundler/src/main/kotlin/Bundler.kt @@ -19,7 +19,7 @@ package org.ossreviewtoolkit.plugins.packagemanagers.bundler -import com.fasterxml.jackson.databind.JsonNode +import com.charleskorn.kaml.Yaml import java.io.ByteArrayOutputStream import java.io.File @@ -29,6 +29,8 @@ import java.net.HttpURLConnection import kotlin.time.measureTime +import kotlinx.serialization.serializer + import org.apache.logging.log4j.kotlin.logger import org.jruby.embed.LocalContextScope @@ -53,12 +55,9 @@ import org.ossreviewtoolkit.model.config.AnalyzerConfiguration import org.ossreviewtoolkit.model.config.PackageManagerConfiguration import org.ossreviewtoolkit.model.config.RepositoryConfiguration import org.ossreviewtoolkit.model.createAndLogIssue -import org.ossreviewtoolkit.model.fromYaml import org.ossreviewtoolkit.model.orEmpty -import org.ossreviewtoolkit.model.yamlMapper import org.ossreviewtoolkit.utils.common.AlphaNumericComparator import org.ossreviewtoolkit.utils.common.collectMessages -import org.ossreviewtoolkit.utils.common.textValueOrEmpty import org.ossreviewtoolkit.utils.ort.HttpDownloadError import org.ossreviewtoolkit.utils.ort.OkHttpClientHelper import org.ossreviewtoolkit.utils.ort.downloadText @@ -85,6 +84,9 @@ private const val BUNDLER_GEM_NAME = "bundler" */ internal const val BUNDLER_LOCKFILE_NAME = "Gemfile.lock" +// TODO: Remove this again once available upstream. +private inline fun Yaml.decodeFromString(string: String): T = decodeFromString(serializer(), string) + private fun runScriptCode(code: String, workingDir: File? = null): String { val bytes = ByteArrayOutputStream() @@ -306,7 +308,7 @@ class Bundler( } private fun getDependencyGroups(workingDir: File): Map> = - runScriptResource(ROOT_DEPENDENCIES_SCRIPT, workingDir).fromYaml() + YAML.decodeFromString(runScriptResource(ROOT_DEPENDENCIES_SCRIPT, workingDir)) private fun resolveGemsInfo(workingDir: File): MutableMap { val stdout = runScriptResource(RESOLVE_DEPENDENCIES_SCRIPT, workingDir) @@ -314,7 +316,8 @@ class Bundler( // The metadata produced by the "resolve_dependencies.rb" script separates specs for packages with the "\0" // character as delimiter. val gemsInfo = stdout.split('\u0000').map { - GemInfo.createFromMetadata(yamlMapper.readTree(it)) + val spec = YAML.decodeFromString(it) + GemInfo.createFromMetadata(spec) }.associateByTo(mutableMapOf()) { it.name } @@ -365,7 +368,8 @@ class Bundler( val url = "https://rubygems.org/api/v2/rubygems/$name/versions/$version.yaml?platform=ruby" return okHttpClient.downloadText(url).mapCatching { - GemInfo.createFromGem(yamlMapper.readTree(it)) + val details = YAML.decodeFromString(it) + GemInfo.createFromGem(details) }.onFailure { val error = (it as? HttpDownloadError) ?: run { logger.warn { "Unable to retrieve metadata for gem '$name' from RubyGems: ${it.message}" } @@ -416,65 +420,58 @@ internal data class GemInfo( val artifact: RemoteArtifact ) { companion object { - fun createFromMetadata(node: JsonNode): GemInfo { - val runtimeDependencies = node["dependencies"]?.asIterable()?.mapNotNull { dependency -> - dependency["name"]?.textValue()?.takeIf { dependency["type"]?.textValue() == ":runtime" } - }?.toSet() + fun createFromMetadata(spec: GemSpec): GemInfo { + val runtimeDependencies = spec.dependencies.mapNotNullTo(mutableSetOf()) { (name, type) -> + name.takeIf { type == VersionDetails.Scope.RUNTIME.toString() } + } + + val homepage = spec.homepage.orEmpty() - val homepage = node["homepage"].textValueOrEmpty() return GemInfo( - node["name"].textValue(), - node["version"]["version"].textValue(), + spec.name, + spec.version.version, homepage, - node["authors"]?.toList().mapToSetOfNotEmptyStrings(), - node["licenses"]?.toList().mapToSetOfNotEmptyStrings(), - node["description"].textValueOrEmpty(), - runtimeDependencies.orEmpty(), + spec.authors.mapToSetOfNotEmptyStrings(), + spec.licenses.mapToSetOfNotEmptyStrings(), + spec.description.orEmpty(), + runtimeDependencies, VcsHost.parseUrl(homepage), RemoteArtifact.EMPTY ) } - fun createFromGem(node: JsonNode): GemInfo { - val runtimeDependencies = node["dependencies"]?.get("runtime")?.mapNotNull { dependency -> - dependency["name"]?.textValue() - }?.toSet() + fun createFromGem(details: VersionDetails): GemInfo { + val runtimeDependencies = details.dependencies[VersionDetails.Scope.RUNTIME] + ?.mapTo(mutableSetOf()) { it.name } + .orEmpty() - val vcs = sequenceOf(node["source_code_uri"], node["homepage_uri"]).mapNotNull { uri -> - uri?.textValue()?.takeIf { it.isNotEmpty() } - }.firstOrNull()?.let { - VcsHost.parseUrl(it) - }.orEmpty() + val vcs = listOfNotNull(details.sourceCodeUri, details.homepageUri) + .mapToSetOfNotEmptyStrings() + .firstOrNull() + ?.let { VcsHost.parseUrl(it) } + .orEmpty() - val artifact = if (node.hasNonNull("gem_uri") && node.hasNonNull("sha")) { - val sha = node["sha"].textValue() - RemoteArtifact(node["gem_uri"].textValue(), Hash.create(sha)) + val artifact = if (details.gemUri != null && details.sha != null) { + RemoteArtifact(details.gemUri, Hash.create(details.sha)) } else { RemoteArtifact.EMPTY } return GemInfo( - node["name"].textValue(), - node["version"].textValue(), - node["homepage_uri"].textValueOrEmpty(), - node["authors"].textValueOrEmpty().split(',').mapToSetOfNotEmptyStrings(), - node["licenses"]?.toList().mapToSetOfNotEmptyStrings(), - node["info"].textValueOrEmpty(), - runtimeDependencies.orEmpty(), + details.name, + details.version, + details.homepageUri.orEmpty(), + details.authors?.split(',').mapToSetOfNotEmptyStrings(), + details.licenses.mapToSetOfNotEmptyStrings(), + details.info.orEmpty(), + runtimeDependencies, vcs, artifact ) } - private inline fun Collection?.mapToSetOfNotEmptyStrings(): Set = - this?.mapNotNullTo(mutableSetOf()) { entry -> - val text = when (T::class) { - JsonNode::class -> (entry as JsonNode).textValue() - else -> entry.toString() - } - - text?.trim()?.takeIf { it.isNotEmpty() } - }.orEmpty() + private fun Collection?.mapToSetOfNotEmptyStrings(): Set = + this?.mapNotNullTo(mutableSetOf()) { string -> string.trim().takeUnless { it.isEmpty() } }.orEmpty() } fun merge(other: GemInfo): GemInfo { diff --git a/plugins/package-managers/bundler/src/main/kotlin/Model.kt b/plugins/package-managers/bundler/src/main/kotlin/Model.kt new file mode 100644 index 000000000000..11ca311acb73 --- /dev/null +++ b/plugins/package-managers/bundler/src/main/kotlin/Model.kt @@ -0,0 +1,145 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.bundler + +import com.charleskorn.kaml.Yaml +import com.charleskorn.kaml.YamlConfiguration +import com.charleskorn.kaml.YamlNamingStrategy + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +internal val YAML = Yaml( + configuration = YamlConfiguration(strictMode = false, yamlNamingStrategy = YamlNamingStrategy.SnakeCase) +) + +/** + * The information for a Gem as typically defined in a ".gemspec" file, see + * - https://guides.rubygems.org/specification-reference/ + * - https://github.com/rubygems/rubygems/blob/95128500fba28e3bb03d3bbfcf8536d00a893035/lib/rubygems/specification.rb#L19-L40 + */ +@Serializable +internal data class GemSpec( + /** The Gem’s name. */ + val name: String, + + /** The Gem’s specified version. */ + val version: Version, + + /** A long description of this Gem. */ + val description: String? = null, + + /** A short summary of this Gem’s description. */ + val summary: String? = null, + + /** The URI of this Gem’s homepage. */ + val homepage: String? = null, + + /** A list of authors for this Gem. */ + val authors: List = emptyList(), + + /** The license(s) for the Gem. Each license must be a short name, no more than 64 characters. */ + val licenses: List = emptyList(), + + /** The list of specified dependencies. */ + val dependencies: List = emptyList() +) { + @Serializable + data class Version( + /** The version string, containing numbers and periods, such as "1.0.0.pre" for a prerelease. */ + val version: String + ) + + @Serializable + data class Dependency( + /** The name of the dependency. */ + val name: String, + + /** The type of the dependency as a Ruby symbol, one of ":development" or ":runtime". */ + val type: String + ) +} + +/** + * Version details for a specific Gem version, see + * - https://guides.rubygems.org/rubygems-org-api-v2/ + * - https://github.com/rubygems/rubygems.org/blob/1f308c8d55403ccc04df407399bcafce87aa5016/app/models/rubygem.rb#L211-L239 + */ +@Serializable +internal data class VersionDetails( + /** The Gem’s name. */ + val name: String, + + /** The version string, containing numbers and periods, such as "1.0.0.pre" for a prerelease. */ + val version: String, + + /** A comma-separated list of authors for this Gem. */ + val authors: String? = null, + + /** A long description of this Gem. */ + val description: String? = null, + + /** A short summary of this Gem’s description. */ + val summary: String? = null, + + /** A synthetic field that contains either the descript or summary, if present. */ + val info: String? = null, + + /** The license(s) for the Gem. Each license must be a short name, no more than 64 characters. */ + val licenses: List = emptyList(), + + /** The SHA256 hash of the Gem artifact. */ + val sha: String? = null, + + /** The download URI of the Gem artifact. */ + val gemUri: String? = null, + + /** The URI of this Gem’s homepage. */ + val homepageUri: String? = null, + + /** The URI of this Gem’s source code. */ + val sourceCodeUri: String? = null, + + /** A map of dependencies per scope. */ + val dependencies: Map> = emptyMap() +) { + @Serializable + enum class Scope { + /** Dependencies required during development. */ + @SerialName("development") + DEVELOPMENT, + + /** Dependencies required at runtime. */ + @SerialName("runtime") + RUNTIME; + + /** A string representation as a Ruby symbol for convenient dependency type matching. */ + override fun toString() = ":${name.lowercase()}" + } + + @Serializable + data class Dependency( + /** The name of the dependency Gem. */ + val name: String, + + /** The version requirements string of the dependency Gem. */ + val requirements: String? = null + ) +} diff --git a/plugins/package-managers/bundler/src/test/kotlin/BundlerTest.kt b/plugins/package-managers/bundler/src/test/kotlin/BundlerTest.kt index bbc6b42e893d..45c34c94cc81 100644 --- a/plugins/package-managers/bundler/src/test/kotlin/BundlerTest.kt +++ b/plugins/package-managers/bundler/src/test/kotlin/BundlerTest.kt @@ -19,6 +19,8 @@ package org.ossreviewtoolkit.plugins.packagemanagers.bundler +import com.charleskorn.kaml.decodeFromStream + import io.kotest.core.spec.style.WordSpec import io.kotest.engine.spec.tempdir import io.kotest.matchers.shouldBe @@ -30,7 +32,6 @@ import org.ossreviewtoolkit.model.HashAlgorithm import org.ossreviewtoolkit.model.RemoteArtifact import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType -import org.ossreviewtoolkit.model.readValue class BundlerTest : WordSpec({ "parseBundlerVersionFromLockfile()" should { @@ -59,8 +60,11 @@ class BundlerTest : WordSpec({ "createFromGem()" should { "parse YAML metadata for a Gem correctly" { val rubyGemsFile = File("src/test/assets/rspec-3.7.0.yaml") + val details = rubyGemsFile.inputStream().use { + YAML.decodeFromStream(it) + } - val gemInfo = GemInfo.createFromGem(rubyGemsFile.readValue()) + val gemInfo = GemInfo.createFromGem(details) gemInfo shouldBe GemInfo( name = "rspec",