Skip to content

Commit

Permalink
refactor(pub): Use a data class for parsing the lockfile
Browse files Browse the repository at this point in the history
Signed-off-by: Frank Viernau <frank_viernau@epam.com>
  • Loading branch information
fviernau committed Aug 30, 2024
1 parent 28c4149 commit d4fd3f1
Show file tree
Hide file tree
Showing 5 changed files with 165 additions and 114 deletions.
1 change: 1 addition & 0 deletions plugins/package-managers/pub/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ dependencies {

implementation(libs.jackson.databind)
implementation(libs.jackson.dataformat.yaml)
implementation(libs.jackson.module.kotlin)

funTestImplementation(projects.plugins.packageManagers.gradlePackageManager)

Expand Down
77 changes: 77 additions & 0 deletions plugins/package-managers/pub/src/main/kotlin/Lockfile.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
* Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.plugins.packagemanagers.pub

import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.DeserializationContext
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import com.fasterxml.jackson.databind.deser.std.StdDeserializer
import com.fasterxml.jackson.module.kotlin.readValue

import java.io.File

import org.ossreviewtoolkit.model.yamlMapper
import org.ossreviewtoolkit.plugins.packagemanagers.pub.PackageInfo.Description

internal fun parseLockfile(lockfile: File) = yamlMapper.readValue<Lockfile>(lockfile)

/**
* See https://github.com/dart-lang/pub/blob/d86e3c979a3889fed61b68dae9f9156d0891704d/lib/src/lock_file.dart#L18.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
internal data class Lockfile(
val packages: Map<String, PackageInfo> = emptyMap()
)

/**
* See https://github.com/dart-lang/pub/blob/d86e3c979a3889fed61b68dae9f9156d0891704d/lib/src/package_name.dart#L73.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
internal data class PackageInfo(
val dependency: String,
@JsonDeserialize(using = DescriptionDeserializer::class)
val description: Description,
val source: String? = null,
val version: String? = null
) {
@JsonIgnoreProperties(ignoreUnknown = true)
data class Description(
val name: String? = null,
val url: String? = null,
val path: String? = null,
@JsonProperty("resolved-ref")
val resolvedRef: String? = null,
val relative: Boolean? = null,
val sha256: String? = null
)
}

internal class DescriptionDeserializer : StdDeserializer<Description>(Description::class.java) {
override fun deserialize(parser: JsonParser, context: DeserializationContext): Description {
val node = context.readTree(parser)
return if (node.isTextual) {
Description(name = node.textValue())
} else {
parser.codec.readValue(node.traverse(), Description::class.java)
}
}
}
52 changes: 26 additions & 26 deletions plugins/package-managers/pub/src/main/kotlin/Pub.kt
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ class Pub(

logger.info { "Reading $PUB_LOCK_FILE file in $workingDir." }

val lockfile = yamlMapper.readTree(workingDir.resolve(PUB_LOCK_FILE))
val lockfile = parseLockfile(workingDir.resolve(PUB_LOCK_FILE))

logger.info { "Successfully read lockfile." }

Expand Down Expand Up @@ -329,7 +329,7 @@ class Pub(
private fun parseScope(
scopeName: String,
manifest: JsonNode,
lockfile: JsonNode,
lockfile: Lockfile,
packages: Map<Identifier, Package>,
labels: Map<String, String>,
workingDir: File
Expand All @@ -346,7 +346,7 @@ class Pub(
private fun buildDependencyTree(
dependencies: List<String>,
manifest: JsonNode,
lockfile: JsonNode,
lockfile: Lockfile,
packages: Map<Identifier, Package>,
labels: Map<String, String>,
workingDir: File,
Expand All @@ -364,16 +364,16 @@ class Pub(
// dependency tree.
if (packageName in processedPackages) return@forEach

val pkgInfoFromLockfile = lockfile["packages"][packageName]
val pkgInfoFromLockfile = lockfile.packages[packageName]
// If the package is marked as SDK (e.g. flutter, flutter_test, dart) we cannot resolve it correctly as
// it is not stored in .pub-cache. For now, we just ignore those SDK packages.
if (pkgInfoFromLockfile == null || pkgInfoFromLockfile["source"].textValueOrEmpty() == "sdk") return@forEach
if (pkgInfoFromLockfile == null || pkgInfoFromLockfile.source == "sdk") return@forEach

val id = Identifier(
type = managerName,
namespace = "",
name = packageName,
version = pkgInfoFromLockfile["version"].textValueOrEmpty()
version = pkgInfoFromLockfile.version.orEmpty()
)

val packageInfo = packages[id] ?: throw IOException("Could not find package info for $packageName")
Expand Down Expand Up @@ -430,11 +430,11 @@ class Pub(
private val analyzerResultCacheAndroid = mutableMapOf<String, List<ProjectAnalyzerResult>>()

private fun analyzeAndroidPackages(
packageInfo: JsonNode,
packageInfo: PackageInfo,
labels: Map<String, String>,
workingDir: File
): List<ProjectAnalyzerResult> {
val packageName = packageInfo["description"]["name"].textValueOrEmpty()
val packageName = packageInfo.description.name.orEmpty()

// We cannot find packages without a valid name.
if (packageName.isEmpty()) return emptyList()
Expand Down Expand Up @@ -469,9 +469,9 @@ class Pub(
}
}

private fun analyzeIosPackages(packageInfo: JsonNode, workingDir: File): ProjectAnalyzerResult? {
private fun analyzeIosPackages(packageInfo: PackageInfo, workingDir: File): ProjectAnalyzerResult? {
// TODO: Implement similar to `scanAndroidPackages` once CocoaPods is implemented.
val packageName = packageInfo["description"]["name"].textValueOrEmpty()
val packageName = packageInfo.description.name.orEmpty()

// We cannot find packages without a valid name.
if (packageName.isEmpty()) return null
Expand Down Expand Up @@ -521,7 +521,7 @@ class Pub(
}

private fun parseInstalledPackages(
lockfile: JsonNode,
lockfile: Lockfile,
labels: Map<String, String>,
workingDir: File
): ParsePackagesResult {
Expand All @@ -533,21 +533,21 @@ class Pub(
// Flag if the project is a Flutter project.
var containsFlutter = false

lockfile["packages"]?.fields()?.forEach { (packageName, pkgInfoFromLockfile) ->
lockfile.packages.forEach { (packageName, pkgInfoFromLockfile) ->
try {
val version = pkgInfoFromLockfile["version"].textValueOrEmpty()
val version = pkgInfoFromLockfile.version.orEmpty()
var description = ""
var rawName = ""
var homepageUrl = ""
var vcs = VcsInfo.EMPTY
var authors = emptySet<String>()

val source = pkgInfoFromLockfile["source"].textValueOrEmpty()
val source = pkgInfoFromLockfile.source.orEmpty()

when {
source == "path" -> {
rawName = packageName
val path = pkgInfoFromLockfile["description"]["path"].textValueOrEmpty()
val path = pkgInfoFromLockfile.description.path.orEmpty()
vcs = VersionControlSystem.forDirectory(workingDir.resolve(path))?.getInfo() ?: run {
logger.warn {
"Invalid path of package $rawName: " +
Expand All @@ -568,9 +568,9 @@ class Pub(

vcs = VcsInfo(
type = VcsType.GIT,
url = normalizeVcsUrl(pkgInfoFromLockfile["description"]["url"].textValueOrEmpty()),
revision = pkgInfoFromLockfile["description"]["resolved-ref"].textValueOrEmpty(),
path = pkgInfoFromLockfile["description"]["path"].textValueOrEmpty()
url = normalizeVcsUrl(pkgInfoFromLockfile.description.url.orEmpty()),
revision = pkgInfoFromLockfile.description.resolvedRef.orEmpty(),
path = pkgInfoFromLockfile.description.path.orEmpty()
)
}

Expand All @@ -591,7 +591,7 @@ class Pub(
vcs = VcsHost.parseUrl(repositoryUrl).copy(revision = "")
}

pkgInfoFromLockfile["description"].textValueOrEmpty() == "flutter" -> {
pkgInfoFromLockfile.description.path.orEmpty() == "flutter" -> {
// Set Flutter flag, which triggers another scan for iOS and Android native dependencies.
containsFlutter = true
// Set hardcoded package details.
Expand All @@ -600,7 +600,7 @@ class Pub(
description = "Flutter SDK"
}

pkgInfoFromLockfile["description"].textValueOrEmpty() == "flutter_test" -> {
pkgInfoFromLockfile.description.path.orEmpty() == "flutter_test" -> {
// Set hardcoded package details.
rawName = "flutter_test"
homepageUrl = "https://github.com/flutter/flutter/tree/master/packages/flutter_test"
Expand All @@ -612,10 +612,10 @@ class Pub(
logger.warn { "No version information found for package $rawName." }
}

val hostUrl = pkgInfoFromLockfile["description"]["url"].textValueOrEmpty()
val hostUrl = pkgInfoFromLockfile.description.url.orEmpty()

val sourceArtifact = if (source == "hosted" && hostUrl.isNotEmpty() && version.isNotEmpty()) {
val sha256 = pkgInfoFromLockfile["description"]["sha256"].textValue()
val sha256 = pkgInfoFromLockfile.description.sha256.orEmpty()

RemoteArtifact(
url = "$hostUrl/packages/$rawName/versions/$version.tar.gz",
Expand Down Expand Up @@ -648,7 +648,7 @@ class Pub(
} catch (e: JacksonYAMLParseException) {
e.showStackTrace()

val packageVersion = pkgInfoFromLockfile["version"].textValueOrEmpty()
val packageVersion = pkgInfoFromLockfile.version
issues += createAndLogIssue(
source = managerName,
message = "Failed to parse $PUBSPEC_YAML for package $packageName:$packageVersion: " +
Expand All @@ -661,7 +661,7 @@ class Pub(
// each Pub dependency manually, as the analyzer will only analyze the projectRoot, but not the packages in
// the ".pub-cache" directory.
if (containsFlutter && !pubDependenciesOnly) {
lockfile["packages"]?.forEach { pkgInfoFromLockfile ->
lockfile.packages.values.forEach { pkgInfoFromLockfile ->
// As this package contains Flutter, trigger Gradle manually for it.
analyzeAndroidPackages(pkgInfoFromLockfile, labels, workingDir).forEach { result ->
result.collectPackagesByScope("releaseCompileClasspath").forEach { pkg ->
Expand All @@ -685,12 +685,12 @@ class Pub(
return ParsePackagesResult(packages, issues)
}

private fun readPackageInfoFromCache(packageInfo: JsonNode, workingDir: File): JsonNode {
private fun readPackageInfoFromCache(packageInfo: PackageInfo, workingDir: File): JsonNode {
val definitionFile = reader.findFile(packageInfo, workingDir, PUBSPEC_YAML)
if (definitionFile == null) {
createAndLogIssue(
source = managerName,
message = "Could not find '$PUBSPEC_YAML' for '${packageInfo["name"].textValueOrEmpty()}'.",
message = "Could not find '$PUBSPEC_YAML' for '${packageInfo.description.name.orEmpty()}'.",
severity = Severity.WARNING
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,14 @@

package org.ossreviewtoolkit.plugins.packagemanagers.pub.utils

import com.fasterxml.jackson.databind.JsonNode

import java.io.File

import org.apache.logging.log4j.kotlin.logger

import org.ossreviewtoolkit.downloader.VcsHost
import org.ossreviewtoolkit.plugins.packagemanagers.pub.PackageInfo
import org.ossreviewtoolkit.utils.common.Os
import org.ossreviewtoolkit.utils.common.isSymbolicLink
import org.ossreviewtoolkit.utils.common.textValueOrEmpty

/**
* A reader for the Pub cache directory. It looks for files in the ".pub-cache" directory in the user's home
Expand All @@ -50,7 +48,7 @@ internal class PubCacheReader(flutterHome: File? = null) {
flutterHome?.resolve(".pub-cache")?.takeIf { it.isDirectory }
}

fun findFile(packageInfo: JsonNode, workingDir: File, filename: String): File? {
fun findFile(packageInfo: PackageInfo, workingDir: File, filename: String): File? {
val artifactRootDir = findProjectRoot(packageInfo, workingDir) ?: return null
// Try to locate the file directly.
val file = artifactRootDir.resolve(filename)
Expand All @@ -62,15 +60,15 @@ internal class PubCacheReader(flutterHome: File? = null) {
.find { !it.isSymbolicLink() && it.isFile && it.name == filename }
}

fun findProjectRoot(packageInfo: JsonNode, workingDir: File): File? {
val packageVersion = packageInfo["version"].textValueOrEmpty()
val type = packageInfo["source"].textValueOrEmpty()
val description = packageInfo["description"]
val packageName = description["name"].textValueOrEmpty()
val url = description["url"].textValueOrEmpty()
val resolvedRef = description["resolved-ref"].textValueOrEmpty()
val resolvedPath = description["path"].textValueOrEmpty()
val isPathRelative = description["relative"]?.booleanValue() == true
fun findProjectRoot(packageInfo: PackageInfo, workingDir: File): File? {
val packageVersion = packageInfo.version.orEmpty()
val type = packageInfo.source.orEmpty()
val description = packageInfo.description
val packageName = description.name.orEmpty()
val url = description.url.orEmpty()
val resolvedRef = description.resolvedRef.orEmpty()
val resolvedPath = description.path.orEmpty()
val isPathRelative = description.relative == true

if (type == "path" && resolvedPath.isNotEmpty()) {
// For "path" packages, the path should be the absolute resolved path
Expand Down
Loading

0 comments on commit d4fd3f1

Please sign in to comment.