From efbba2e15a029f485494e2573d1b1bd1a8a48162 Mon Sep 17 00:00:00 2001 From: Jude Niroshan Date: Tue, 7 May 2024 14:36:11 +0200 Subject: [PATCH 1/6] feat: introduce new flag to avoid using pip freeze and pip show Signed-off-by: Jude Niroshan --- README.md | 5 + .../exhort/providers/PythonPipProvider.java | 36 +- .../exhort/utils/PythonControllerBase.java | 355 ++++++++------ .../redhat/exhort/utils/PythonDependency.java | 56 +++ src/main/java/module-info.java | 2 + .../providers/Python_Provider_Test.java | 36 ++ .../tst_manifests/pip/pipdeptree.json | 432 ++++++++++++++++++ 7 files changed, 760 insertions(+), 162 deletions(-) create mode 100644 src/main/java/com/redhat/exhort/utils/PythonDependency.java create mode 100644 src/test/resources/tst_manifests/pip/pipdeptree.json diff --git a/README.md b/README.md index df409ec..14e2d6f 100644 --- a/README.md +++ b/README.md @@ -467,6 +467,11 @@ A New setting is introduced - `EXHORT_PYTHON_INSTALL_BEST_EFFORTS` (as both env 1. `EXHORT_PYTHON_INSTALL_BEST_EFFORTS`="false" - install requirements.txt while respecting declared versions for all packages. 2. `EXHORT_PYTHON_INSTALL_BEST_EFFORTS`="true" - install all packages from requirements.txt, not respecting the declared version, but trying to install a version tailored for the used python version, when using this setting,you must set setting `MATCH_MANIFEST_VERSIONS`="false" +##### Using `pipdeptree` +By Default, The API algorithm will use native commands of PIP installer as data source to build the dependency tree. +It's also possible, to use lightweight Python PIP utility [pipdeptree](https://pypi.org/project/pipdeptree/) as data source instead, in order to activate this, +Need to set environment variable/option - `EXHORT_PIP_USE_DEP_TREE` to true. + ### Image Support Generate vulnerability analysis report for container images. diff --git a/src/main/java/com/redhat/exhort/providers/PythonPipProvider.java b/src/main/java/com/redhat/exhort/providers/PythonPipProvider.java index e94125c..c9bdce0 100644 --- a/src/main/java/com/redhat/exhort/providers/PythonPipProvider.java +++ b/src/main/java/com/redhat/exhort/providers/PythonPipProvider.java @@ -79,11 +79,9 @@ public Content provideStack(Path manifestPath) throws IOException { printDependenciesTree(dependencies); Sbom sbom = SbomFactory.newInstance(Sbom.BelongingCondition.PURL, "sensitive"); sbom.addRoot(toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION)); - dependencies.stream() - .forEach( - (component) -> { - addAllDependencies(sbom.getRoot(), component, sbom); - }); + for (Map component : dependencies) { + addAllDependencies(sbom.getRoot(), component, sbom); + } byte[] requirementsFile = Files.readAllBytes(manifestPath); handleIgnoredDependencies(new String(requirementsFile), sbom); return new Content( @@ -92,25 +90,17 @@ public Content provideStack(Path manifestPath) throws IOException { private void addAllDependencies(PackageURL source, Map component, Sbom sbom) { - sbom.addDependency( - source, toPurl((String) component.get("name"), (String) component.get("version"))); - List directDeps = (List) component.get("dependencies"); - if (directDeps != null) - // { - directDeps.stream() - .forEach( - dep -> { - String name = (String) dep.get("name"); - String version = (String) dep.get("version"); - - addAllDependencies( - toPurl((String) component.get("name"), (String) component.get("version")), - dep, - sbom); - }); - // - // } + PackageURL packageURL = + toPurl((String) component.get("name"), (String) component.get("version")); + sbom.addDependency(source, packageURL); + List> directDeps = + (List>) component.get("dependencies"); + if (directDeps != null) { + for (Map dep : directDeps) { + addAllDependencies(packageURL, dep, sbom); + } + } } @Override diff --git a/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java b/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java index 4dd1834..9afabef 100644 --- a/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java +++ b/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java @@ -18,7 +18,9 @@ import static com.redhat.exhort.impl.ExhortApi.*; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.redhat.exhort.exception.PackageNotInstalledException; import com.redhat.exhort.logging.LoggersFactory; import com.redhat.exhort.tools.Operations; @@ -203,14 +205,12 @@ public final List> getDependencies( if (automaticallyInstallPackageOnEnvironment()) { boolean installBestEfforts = getBooleanValueEnvironment("EXHORT_PYTHON_INSTALL_BEST_EFFORTS", "false"); - // make best efforts to install the requirements.txt on the virtual environment created from - // the python3 - // passed in. - // that means that it will install the packages without referring to the versions, but will - // let pip choose - // the version - // tailored for version of the python environment( and of pip package manager) for each - // package. + /* + make best efforts to install the requirements.txt on the virtual environment created from + the python3 passed in. that means that it will install the packages without referring to + the versions, but will let pip choose the version tailored for version of the python + environment( and of pip package manager) for each package. + */ if (installBestEfforts) { boolean matchManifestVersions = getBooleanValueEnvironment("MATCH_MANIFEST_VERSIONS", "true"); @@ -221,9 +221,7 @@ public final List> getDependencies( } else { installingRequirementsOneByOne(pathToRequirements); } - } - // - else { + } else { installPackages(pathToRequirements); } } @@ -268,34 +266,10 @@ private void installingRequirementsOneByOne(String pathToRequirements) { private List> getDependenciesImpl( String pathToRequirements, boolean includeTransitive) { List> dependencies = new ArrayList<>(); - String freeze = getPipFreezeFromEnvironment(); - String freezeMessage = ""; - if (debugLoggingIsNeeded()) { - freezeMessage = - String.format( - "Package Manager PIP freeze --all command result output -> %s %s", - System.lineSeparator(), freeze); - log.info(freezeMessage); - } - String[] deps = freeze.split(System.lineSeparator()); - String depNames = - Arrays.stream(deps) - .map(PythonControllerBase::getDependencyName) - .collect(Collectors.joining(" ")); - String pipShowOutput = getPipShowFromEnvironment(depNames); - if (debugLoggingIsNeeded()) { - String pipShowMessage = - String.format( - "Package Manager PIP show command result output -> %s %s", - System.lineSeparator(), pipShowOutput); - log.info(pipShowMessage); - } - List allPipShowLines = splitPipShowLines(pipShowOutput); - boolean matchManifestVersions = getBooleanValueEnvironment("MATCH_MANIFEST_VERSIONS", "true"); - Map CachedTree = new HashMap<>(); + Map cachedEnvironmentDeps = new HashMap<>(); + fillCacheWithEnvironmentDeps(cachedEnvironmentDeps); List linesOfRequirements; try { - linesOfRequirements = Files.readAllLines(Path.of(pathToRequirements)).stream() .filter((line) -> !line.startsWith("#")) @@ -304,96 +278,154 @@ private List> getDependenciesImpl( } catch (IOException e) { throw new RuntimeException(e); } - allPipShowLines.stream() - .forEach( - record -> { - String dependencyNameShow = getDependencyNameShow(record); - StringInsensitive stringInsensitive = new StringInsensitive(dependencyNameShow); - CachedTree.put(stringInsensitive, record); - CachedTree.putIfAbsent( - new StringInsensitive(dependencyNameShow.replace("-", "_")), record); - CachedTree.putIfAbsent( - new StringInsensitive(dependencyNameShow.replace("_", "-")), record); - }); - ObjectMapper om = new ObjectMapper(); - String tree; try { - tree = om.writerWithDefaultPrettyPrinter().writeValueAsString(CachedTree); + ObjectMapper om = new ObjectMapper(); + om.writerWithDefaultPrettyPrinter().writeValueAsString(cachedEnvironmentDeps); } catch (JsonProcessingException e) { throw new RuntimeException(e); } - linesOfRequirements.stream() - .forEach( - dep -> { - if (matchManifestVersions) { - String dependencyName; - String manifestVersion; - String installedVersion = ""; - int doubleEqualSignPosition; - if (dep.contains("==")) { - doubleEqualSignPosition = dep.indexOf("=="); - manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim(); - if (manifestVersion.contains("#")) { - var hashCharIndex = manifestVersion.indexOf("#"); - manifestVersion = manifestVersion.substring(0, hashCharIndex); - } - dependencyName = getDependencyName(dep); - String pipShowRecord = CachedTree.get(new StringInsensitive(dependencyName)); - if (pipShowRecord != null) { - installedVersion = getDependencyVersion(pipShowRecord); - } - if (!installedVersion.trim().equals("")) { - if (!manifestVersion.trim().equals(installedVersion.trim())) { - throw new RuntimeException( - String.format( - "Can't continue with analysis - versions mismatch for dependency" - + " name=%s, manifest version=%s, installed Version=%s, if you" - + " want to allow version mismatch for analysis between installed" - + " and requested packages, set environment variable/setting -" - + " MATCH_MANIFEST_VERSIONS=false", - dependencyName, manifestVersion, installedVersion)); - } - } - } - } - List path = new ArrayList<>(); - String depName = getDependencyName(dep.toLowerCase()); - path.add(depName); - bringAllDependencies(dependencies, depName, CachedTree, includeTransitive, path); - }); + boolean matchManifestVersions = getBooleanValueEnvironment("MATCH_MANIFEST_VERSIONS", "true"); + + for (String dep : linesOfRequirements) { + if (matchManifestVersions) { + String dependencyName; + String manifestVersion; + String installedVersion = ""; + int doubleEqualSignPosition; + if (dep.contains("==")) { + doubleEqualSignPosition = dep.indexOf("=="); + manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim(); + if (manifestVersion.contains("#")) { + var hashCharIndex = manifestVersion.indexOf("#"); + manifestVersion = manifestVersion.substring(0, hashCharIndex); + } + dependencyName = getDependencyName(dep); + PythonDependency pythonDependency = + cachedEnvironmentDeps.get(new StringInsensitive(dependencyName)); + if (pythonDependency != null) { + installedVersion = pythonDependency.getVersion(); + } + if (!installedVersion.trim().equals("")) { + if (!manifestVersion.trim().equals(installedVersion.trim())) { + throw new RuntimeException( + String.format( + "Can't continue with analysis - versions mismatch for dependency" + + " name=%s, manifest version=%s, installed Version=%s, if you" + + " want to allow version mismatch for analysis between installed" + + " and requested packages, set environment variable/setting -" + + " MATCH_MANIFEST_VERSIONS=false", + dependencyName, manifestVersion, installedVersion)); + } + } + } + } + List path = new ArrayList<>(); + String selectedDepName = getDependencyName(dep.toLowerCase()); + path.add(selectedDepName); + bringAllDependencies( + dependencies, selectedDepName, cachedEnvironmentDeps, includeTransitive, path); + } return dependencies; } private String getPipShowFromEnvironment(String depNames) { - return getPipCommandInvokedOrDecodedFromEnvironment( - "EXHORT_PIP_SHOW", pipBinaryLocation, "show", depNames); - // return Operations.runProcessGetOutput(pythonEnvironmentDir, pipBinaryLocation, "show", - // depNames); + return executeCommandOrExtractFromEnv("EXHORT_PIP_SHOW", pipBinaryLocation, "show", depNames); } String getPipFreezeFromEnvironment() { - return getPipCommandInvokedOrDecodedFromEnvironment( + return executeCommandOrExtractFromEnv( "EXHORT_PIP_FREEZE", pipBinaryLocation, "freeze", "--all"); } - private String getPipCommandInvokedOrDecodedFromEnvironment(String EnvVar, String... cmdList) { - return getStringValueEnvironment(EnvVar, "").trim().equals("") - ? Operations.runProcessGetOutput(pythonEnvironmentDir, cmdList) - : new String(Base64.getDecoder().decode(getStringValueEnvironment(EnvVar, ""))); + List getDependencyTreeJsonFromPipDepTree() { + executeCommandOrExtractFromEnv( + "EXHORT_PIP_PIPDEPTREE", pipBinaryLocation, "install", "pipdeptree"); + + String pipdeptreeJsonString; + if (isVirtualEnv()) { + pipdeptreeJsonString = + executeCommandOrExtractFromEnv("EXHORT_PIP_PIPDEPTREE", "./bin/pipdeptree", "--json"); + } else if (isRealEnv()) { + pipdeptreeJsonString = + executeCommandOrExtractFromEnv( + "EXHORT_PIP_PIPDEPTREE", pathToPythonBin, "-m", "pipdeptree", "--json"); + } else { + pipdeptreeJsonString = + executeCommandOrExtractFromEnv( + "EXHORT_PIP_PIPDEPTREE", "./bin/pipdeptree", "--json", "--python", pathToPythonBin); + } + if (debugLoggingIsNeeded()) { + String pipdeptreeMessage = + String.format( + "Package Manager pipdeptree --json command result output -> %s %s", + System.lineSeparator(), pipdeptreeJsonString); + log.info(pipdeptreeMessage); + } + return mapToPythonDependencies(pipdeptreeJsonString); + } + + public static List mapToPythonDependencies(String jsonString) { + // Parse JSON string using ObjectMapper + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = null; + try { + rootNode = mapper.readTree(jsonString); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + List dependencies = new ArrayList<>(); + + // Check if it's an array + if (rootNode.isArray()) { + for (JsonNode dependencyNode : rootNode) { + if (dependencyNode.isObject()) { + ObjectNode dependencyObject = (ObjectNode) dependencyNode; + + // Extract information from the nested "package" object + JsonNode packageNode = dependencyObject.get("package"); + String name = packageNode.get("package_name").asText(); + String version = packageNode.get("installed_version").asText(); + + // Extract dependencies (might be an array or an empty object) + JsonNode dependenciesElement = dependencyObject.get("dependencies"); + List depList = new ArrayList<>(); + if (dependenciesElement.isArray()) { + // Loop through the dependencies array and add names + for (JsonNode depNode : dependenciesElement) { + String depName = depNode.get("package_name").asText(); + depList.add(depName); + } + } + + // Create a PythonDependency object and add it to the list + PythonDependency dependency = new PythonDependency(name, version, depList); + dependencies.add(dependency); + } + } + } + + return dependencies; + } + + private String executeCommandOrExtractFromEnv(String EnvVar, String... cmdList) { + String envValue = getStringValueEnvironment(EnvVar, ""); + if (envValue.trim().equals("")) + return Operations.runProcessGetOutput(pythonEnvironmentDir, cmdList); + return new String(Base64.getDecoder().decode(envValue)); } private void bringAllDependencies( - List> dependencies, + List> dependencyList, String depName, - Map cachedTree, + Map cachedTree, boolean includeTransitive, List path) { - if (dependencies == null || depName.trim().equals("")) return; + if (dependencyList == null || depName.trim().equals("")) return; - String record = cachedTree.get(new StringInsensitive(depName)); - if (record == null) { + PythonDependency pythonDependency = cachedTree.get(new StringInsensitive(depName)); + if (pythonDependency == null) { throw new PackageNotInstalledException( String.format( "Package name=>%s is not installed on your python environment, either install it (" @@ -402,42 +434,39 @@ private void bringAllDependencies( + " virtual environment ( will slow down the analysis)", depName)); } - String depVersion = getDependencyVersion(record); - List directDeps = getDepsList(record); - getDependencyNameShow(record); - Map entry = new HashMap(); - dependencies.add(entry); - entry.put("name", getDependencyNameShow(record)); - entry.put("version", depVersion); - List> targetDeps = new ArrayList<>(); - directDeps.stream() - .forEach( - dep -> { - if (!path.contains(dep.toLowerCase())) { - List depList = new ArrayList(); - depList.add(dep.toLowerCase()); - - if (includeTransitive) { - bringAllDependencies( - targetDeps, - dep, - cachedTree, - includeTransitive, - Stream.concat(path.stream(), depList.stream()).collect(Collectors.toList())); - } - } - Collections.sort( - targetDeps, - (o1, o2) -> { - String string1 = (String) (o1.get("name")); - String string2 = (String) (o2.get("name")); - return Arrays.compare(string1.toCharArray(), string2.toCharArray()); - }); - entry.put("dependencies", targetDeps); - }); + + Map dataMap = new HashMap<>(); + dataMap.put("name", pythonDependency.getName()); + dataMap.put("version", pythonDependency.getVersion()); + dependencyList.add(dataMap); + + List> transitiveDepList = new ArrayList<>(); + List directDeps = pythonDependency.getDependencies(); + for (String directDep : directDeps) { + if (!path.contains(directDep.toLowerCase())) { + List depList = new ArrayList<>(); + depList.add(directDep.toLowerCase()); + + if (includeTransitive) { + bringAllDependencies( + transitiveDepList, + directDep, + cachedTree, + true, + Stream.concat(path.stream(), depList.stream()).collect(Collectors.toList())); + } + } + transitiveDepList.sort( + (map1, map2) -> { + String string1 = (String) (map1.get("name")); + String string2 = (String) (map2.get("name")); + return Arrays.compare(string1.toCharArray(), string2.toCharArray()); + }); + dataMap.put("dependencies", transitiveDepList); + } } - protected List getDepsList(String pipShowOutput) { + protected List getDepsList(String pipShowOutput) { int requiresKeyIndex = pipShowOutput.indexOf("Requires:"); String requiresToken = pipShowOutput.substring(requiresKeyIndex + 9); int endOfLine = requiresToken.indexOf(System.lineSeparator()); @@ -476,7 +505,6 @@ public static String getDependencyName(String dep) { if (rightTriangleBracket == -1 && leftTriangleBracket == -1 && equalsSign == -1) { depName = dep; } else { - depName = dep.substring(0, minimumIndex); } return depName.trim(); @@ -499,4 +527,53 @@ static List splitPipShowLines(String pipShowOutput) { pipShowOutput.split(System.lineSeparator() + "---" + System.lineSeparator())) .collect(Collectors.toList()); } + + private PythonDependency getPythonDependencyByShowStringBlock(String pipShowStringBlock) { + return new PythonDependency( + getDependencyNameShow(pipShowStringBlock), + getDependencyVersion(pipShowStringBlock), + getDepsList(pipShowStringBlock)); + } + + private void fillCacheWithEnvironmentDeps(Map cache) { + boolean usePipDepTree = getBooleanValueEnvironment("EXHORT_PIP_USE_DEP_TREE", "false"); + if (usePipDepTree) { + getDependencyTreeJsonFromPipDepTree().forEach(d -> saveToCacheWithKeyVariations(cache, d)); + } else { + String freezeOutput = getPipFreezeFromEnvironment(); + if (debugLoggingIsNeeded()) { + String freezeMessage = + String.format( + "Package Manager PIP freeze --all command result output -> %s %s", + System.lineSeparator(), freezeOutput); + log.info(freezeMessage); + } + String[] deps = freezeOutput.split(System.lineSeparator()); + String depNames = + Arrays.stream(deps) + .map(PythonControllerBase::getDependencyName) + .collect(Collectors.joining(" ")); + String pipShowOutput = getPipShowFromEnvironment(depNames); + if (debugLoggingIsNeeded()) { + String pipShowMessage = + String.format( + "Package Manager PIP show command result output -> %s %s", + System.lineSeparator(), pipShowOutput); + log.info(pipShowMessage); + } + splitPipShowLines(pipShowOutput).stream() + .map(this::getPythonDependencyByShowStringBlock) + .forEach(d -> saveToCacheWithKeyVariations(cache, d)); + } + } + + private void saveToCacheWithKeyVariations( + Map cache, PythonDependency pythonDependency) { + StringInsensitive stringInsensitive = new StringInsensitive(pythonDependency.getName()); + cache.put(stringInsensitive, pythonDependency); + cache.putIfAbsent( + new StringInsensitive(pythonDependency.getName().replace("-", "_")), pythonDependency); + cache.putIfAbsent( + new StringInsensitive(pythonDependency.getName().replace("_", "-")), pythonDependency); + } } diff --git a/src/main/java/com/redhat/exhort/utils/PythonDependency.java b/src/main/java/com/redhat/exhort/utils/PythonDependency.java new file mode 100644 index 0000000..e553b2d --- /dev/null +++ b/src/main/java/com/redhat/exhort/utils/PythonDependency.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2023 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.redhat.exhort.utils; + +import java.util.List; + +public class PythonDependency { + String name = ""; + String version = ""; + List dependencies; + + public PythonDependency() {} + + public PythonDependency(String name, String version, List dependencies) { + this.name = name; + this.version = version; + this.dependencies = dependencies; + } + + public String getName() { + return name; + } + + public String getVersion() { + return version; + } + + public List getDependencies() { + return dependencies; + } + + public void setName(String name) { + this.name = name; + } + + public void setVersion(String version) { + this.version = version; + } + + public void setDependencies(List dependencies) { + this.dependencies = dependencies; + } +} diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 91a436a..0b7515d 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -22,6 +22,8 @@ exports com.redhat.exhort.sbom; exports com.redhat.exhort.tools; + opens com.redhat.exhort.utils to + com.fasterxml.jackson.databind; opens com.redhat.exhort.sbom to com.fasterxml.jackson.databind, packageurl.java; diff --git a/src/test/java/com/redhat/exhort/providers/Python_Provider_Test.java b/src/test/java/com/redhat/exhort/providers/Python_Provider_Test.java index 9ca72a0..23411a8 100644 --- a/src/test/java/com/redhat/exhort/providers/Python_Provider_Test.java +++ b/src/test/java/com/redhat/exhort/providers/Python_Provider_Test.java @@ -149,6 +149,42 @@ void test_the_provideStack_with_properties(String testFolder) assertThat(dropIgnored(new String(content.buffer))).isEqualTo(dropIgnored(expectedSbom)); } + @ParameterizedTest + @MethodSource("testFolders") + void test_the_provideStack_with_pipdeptree(String testFolder) + throws IOException, InterruptedException { + // create temp file hosting our sut package.json + System.setProperty("EXHORT_PIP_USE_DEP_TREE", "true"); + var tmpPythonModuleDir = Files.createTempDirectory("exhort_test_"); + var tmpPythonFile = Files.createFile(tmpPythonModuleDir.resolve("requirements.txt")); + try (var is = + getResourceAsStreamDecision( + this.getClass(), + new String[] {"tst_manifests", "pip", testFolder, "requirements.txt"})) { + Files.write(tmpPythonFile, is.readAllBytes()); + } + // load expected SBOM + String expectedSbom; + try (var is = + getResourceAsStreamDecision( + this.getClass(), + new String[] {"tst_manifests", "pip", testFolder, "expected_stack_sbom.json"})) { + expectedSbom = new String(is.readAllBytes()); + } + // when providing stack content for our pom + var content = this.pythonPipProvider.provideStack(tmpPythonFile); + String pipdeptreeContent = this.getStringFromFile("tst_manifests", "pip", "pipdeptree.json"); + String base64Pipdeptree = new String(Base64.getEncoder().encode(pipdeptreeContent.getBytes())); + System.setProperty("EXHORT_PIP_PIPDEPTREE", base64Pipdeptree); + // cleanup + Files.deleteIfExists(tmpPythonFile); + Files.deleteIfExists(tmpPythonModuleDir); + System.clearProperty("EXHORT_PIP_USE_DEP_TREE"); + // verify expected SBOM is returned + assertThat(content.type).isEqualTo(Api.CYCLONEDX_MEDIA_TYPE); + assertThat(dropIgnored(new String(content.buffer))).isEqualTo(dropIgnored(expectedSbom)); + } + @ParameterizedTest @MethodSource("testFolders") void test_the_provideComponent_with_properties(String testFolder) diff --git a/src/test/resources/tst_manifests/pip/pipdeptree.json b/src/test/resources/tst_manifests/pip/pipdeptree.json new file mode 100644 index 0000000..ec01061 --- /dev/null +++ b/src/test/resources/tst_manifests/pip/pipdeptree.json @@ -0,0 +1,432 @@ +[ + { + "package": { + "key": "altgraph", + "package_name": "altgraph", + "installed_version": "0.17.2" + }, + "dependencies": [] + }, + { + "package": { + "key": "anyio", + "package_name": "anyio", + "installed_version": "3.6.2" + }, + "dependencies": [ + { + "key": "idna", + "package_name": "idna", + "installed_version": "2.10", + "required_version": ">=2.8" + }, + { + "key": "sniffio", + "package_name": "sniffio", + "installed_version": "1.2.0", + "required_version": ">=1.1" + } + ] + }, + { + "package": { + "key": "asgiref", + "package_name": "asgiref", + "installed_version": "3.4.1" + }, + "dependencies": [] + }, + { + "package": { + "key": "beautifulsoup4", + "package_name": "beautifulsoup4", + "installed_version": "4.12.2" + }, + "dependencies": [ + { + "key": "soupsieve", + "package_name": "soupsieve", + "installed_version": "2.3.2.post1", + "required_version": ">1.2" + } + ] + }, + { + "package": { + "key": "certifi", + "package_name": "certifi", + "installed_version": "2023.7.22" + }, + "dependencies": [] + }, + { + "package": { + "key": "chardet", + "package_name": "chardet", + "installed_version": "4.0.0" + }, + "dependencies": [] + }, + { + "package": { + "key": "click", + "package_name": "click", + "installed_version": "8.0.4" + }, + "dependencies": [] + }, + { + "package": { + "key": "contextlib2", + "package_name": "contextlib2", + "installed_version": "21.6.0" + }, + "dependencies": [] + }, + { + "package": { + "key": "fastapi", + "package_name": "fastapi", + "installed_version": "0.75.1" + }, + "dependencies": [ + { + "key": "pydantic", + "package_name": "pydantic", + "installed_version": "1.9.2", + "required_version": ">=1.6.2,<2.0.0,!=1.8.1,!=1.8,!=1.7.3,!=1.7.2,!=1.7.1,!=1.7" + }, + { + "key": "starlette", + "package_name": "starlette", + "installed_version": "0.17.1", + "required_version": "==0.17.1" + } + ] + }, + { + "package": { + "key": "flask", + "package_name": "Flask", + "installed_version": "2.0.3" + }, + "dependencies": [ + { + "key": "click", + "package_name": "click", + "installed_version": "8.0.4", + "required_version": ">=7.1.2" + }, + { + "key": "itsdangerous", + "package_name": "itsdangerous", + "installed_version": "2.0.1", + "required_version": ">=2.0" + }, + { + "key": "jinja2", + "package_name": "Jinja2", + "installed_version": "3.0.3", + "required_version": ">=3.0" + }, + { + "key": "werkzeug", + "package_name": "Werkzeug", + "installed_version": "2.0.3", + "required_version": ">=2.0" + } + ] + }, + { + "package": { + "key": "future", + "package_name": "future", + "installed_version": "0.18.2" + }, + "dependencies": [] + }, + { + "package": { + "key": "h11", + "package_name": "h11", + "installed_version": "0.13.0" + }, + "dependencies": [] + }, + { + "package": { + "key": "idna", + "package_name": "idna", + "installed_version": "2.10" + }, + "dependencies": [] + }, + { + "package": { + "key": "immutables", + "package_name": "immutables", + "installed_version": "0.19" + }, + "dependencies": [] + }, + { + "package": { + "key": "importlib-metadata", + "package_name": "importlib-metadata", + "installed_version": "4.8.3" + }, + "dependencies": [ + { + "key": "zipp", + "package_name": "zipp", + "installed_version": "3.6.0", + "required_version": ">=0.5" + } + ] + }, + { + "package": { + "key": "itsdangerous", + "package_name": "itsdangerous", + "installed_version": "2.0.1" + }, + "dependencies": [] + }, + { + "package": { + "key": "jinja2", + "package_name": "Jinja2", + "installed_version": "3.0.3" + }, + "dependencies": [ + { + "key": "markupsafe", + "package_name": "MarkupSafe", + "installed_version": "2.0.1", + "required_version": ">=2.0" + } + ] + }, + { + "package": { + "key": "macholib", + "package_name": "macholib", + "installed_version": "1.15.2" + }, + "dependencies": [ + { + "key": "altgraph", + "package_name": "altgraph", + "installed_version": "0.17.2", + "required_version": ">=0.15" + } + ] + }, + { + "package": { + "key": "markupsafe", + "package_name": "MarkupSafe", + "installed_version": "2.0.1" + }, + "dependencies": [] + }, + { + "package": { + "key": "packaging", + "package_name": "packaging", + "installed_version": "24.0" + }, + "dependencies": [] + }, + { + "package": { + "key": "pip", + "package_name": "pip", + "installed_version": "24.0" + }, + "dependencies": [] + }, + { + "package": { + "key": "pipdeptree", + "package_name": "pipdeptree", + "installed_version": "2.19.1" + }, + "dependencies": [ + { + "key": "packaging", + "package_name": "packaging", + "installed_version": "24.0", + "required_version": ">=23.1" + }, + { + "key": "pip", + "package_name": "pip", + "installed_version": "24.0", + "required_version": ">=23.1.2" + } + ] + }, + { + "package": { + "key": "pydantic", + "package_name": "pydantic", + "installed_version": "1.9.2" + }, + "dependencies": [ + { + "key": "typing-extensions", + "package_name": "typing_extensions", + "installed_version": "4.1.1", + "required_version": ">=3.7.4.3" + } + ] + }, + { + "package": { + "key": "requests", + "package_name": "requests", + "installed_version": "2.25.1" + }, + "dependencies": [ + { + "key": "certifi", + "package_name": "certifi", + "installed_version": "2023.7.22", + "required_version": ">=2017.4.17" + }, + { + "key": "chardet", + "package_name": "chardet", + "installed_version": "4.0.0", + "required_version": ">=3.0.2,<5" + }, + { + "key": "idna", + "package_name": "idna", + "installed_version": "2.10", + "required_version": ">=2.5,<3" + }, + { + "key": "urllib3", + "package_name": "urllib3", + "installed_version": "1.26.16", + "required_version": ">=1.21.1,<1.27" + } + ] + }, + { + "package": { + "key": "setuptools", + "package_name": "setuptools", + "installed_version": "58.0.4" + }, + "dependencies": [] + }, + { + "package": { + "key": "six", + "package_name": "six", + "installed_version": "1.16.0" + }, + "dependencies": [] + }, + { + "package": { + "key": "sniffio", + "package_name": "sniffio", + "installed_version": "1.2.0" + }, + "dependencies": [] + }, + { + "package": { + "key": "soupsieve", + "package_name": "soupsieve", + "installed_version": "2.3.2.post1" + }, + "dependencies": [] + }, + { + "package": { + "key": "starlette", + "package_name": "starlette", + "installed_version": "0.17.1" + }, + "dependencies": [ + { + "key": "anyio", + "package_name": "anyio", + "installed_version": "3.6.2", + "required_version": ">=3.0.0,<4" + } + ] + }, + { + "package": { + "key": "typing-extensions", + "package_name": "typing_extensions", + "installed_version": "4.1.1" + }, + "dependencies": [] + }, + { + "package": { + "key": "urllib3", + "package_name": "urllib3", + "installed_version": "1.26.16" + }, + "dependencies": [] + }, + { + "package": { + "key": "uvicorn", + "package_name": "uvicorn", + "installed_version": "0.17.0" + }, + "dependencies": [ + { + "key": "asgiref", + "package_name": "asgiref", + "installed_version": "3.4.1", + "required_version": ">=3.4.0" + }, + { + "key": "click", + "package_name": "click", + "installed_version": "8.0.4", + "required_version": ">=7.0" + }, + { + "key": "h11", + "package_name": "h11", + "installed_version": "0.13.0", + "required_version": ">=0.8" + } + ] + }, + { + "package": { + "key": "werkzeug", + "package_name": "Werkzeug", + "installed_version": "2.0.3" + }, + "dependencies": [] + }, + { + "package": { + "key": "wheel", + "package_name": "wheel", + "installed_version": "0.37.0" + }, + "dependencies": [] + }, + { + "package": { + "key": "zipp", + "package_name": "zipp", + "installed_version": "3.6.0" + }, + "dependencies": [] + } +] From d57234020917b248613b7561a995d8c31f4876b9 Mon Sep 17 00:00:00 2001 From: Jude Niroshan Date: Wed, 8 May 2024 11:53:16 +0200 Subject: [PATCH 2/6] refactor: use java streams Signed-off-by: Jude Niroshan --- .../exhort/utils/PythonControllerBase.java | 58 +++++++------------ 1 file changed, 21 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java b/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java index 9afabef..d8d7f18 100644 --- a/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java +++ b/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.redhat.exhort.exception.PackageNotInstalledException; import com.redhat.exhort.logging.LoggersFactory; import com.redhat.exhort.tools.Operations; @@ -366,46 +365,31 @@ List getDependencyTreeJsonFromPipDepTree() { } public static List mapToPythonDependencies(String jsonString) { - // Parse JSON string using ObjectMapper - ObjectMapper mapper = new ObjectMapper(); - JsonNode rootNode = null; try { - rootNode = mapper.readTree(jsonString); + // Parse JSON and store in a list of JsonNodes + List jsonNodeList = new ArrayList<>(); + new ObjectMapper().readTree(jsonString).elements().forEachRemaining(jsonNodeList::add); + + return jsonNodeList.stream() + .filter(JsonNode::isObject) + .map( + dependencyNode -> { + String name = dependencyNode.get("package").get("package_name").asText(); + String version = dependencyNode.get("package").get("installed_version").asText(); + + // Extract dependencies + List depList = new ArrayList<>(); + dependencyNode + .get("dependencies") + .elements() + .forEachRemaining(e -> depList.add(e.get("package_name").asText())); + + return new PythonDependency(name, version, depList); + }) + .collect(Collectors.toList()); } catch (JsonProcessingException e) { throw new RuntimeException(e); } - List dependencies = new ArrayList<>(); - - // Check if it's an array - if (rootNode.isArray()) { - for (JsonNode dependencyNode : rootNode) { - if (dependencyNode.isObject()) { - ObjectNode dependencyObject = (ObjectNode) dependencyNode; - - // Extract information from the nested "package" object - JsonNode packageNode = dependencyObject.get("package"); - String name = packageNode.get("package_name").asText(); - String version = packageNode.get("installed_version").asText(); - - // Extract dependencies (might be an array or an empty object) - JsonNode dependenciesElement = dependencyObject.get("dependencies"); - List depList = new ArrayList<>(); - if (dependenciesElement.isArray()) { - // Loop through the dependencies array and add names - for (JsonNode depNode : dependenciesElement) { - String depName = depNode.get("package_name").asText(); - depList.add(depName); - } - } - - // Create a PythonDependency object and add it to the list - PythonDependency dependency = new PythonDependency(name, version, depList); - dependencies.add(dependency); - } - } - } - - return dependencies; } private String executeCommandOrExtractFromEnv(String EnvVar, String... cmdList) { From 2f6e193a4944eb722f48c10b0024455de5c2113b Mon Sep 17 00:00:00 2001 From: Jude Niroshan Date: Wed, 8 May 2024 12:17:46 +0200 Subject: [PATCH 3/6] refactor: rename variable to targetDeps Signed-off-by: Jude Niroshan --- .../com/redhat/exhort/utils/PythonControllerBase.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java b/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java index d8d7f18..6392dd7 100644 --- a/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java +++ b/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java @@ -424,7 +424,7 @@ private void bringAllDependencies( dataMap.put("version", pythonDependency.getVersion()); dependencyList.add(dataMap); - List> transitiveDepList = new ArrayList<>(); + List> targetDeps = new ArrayList<>(); List directDeps = pythonDependency.getDependencies(); for (String directDep : directDeps) { if (!path.contains(directDep.toLowerCase())) { @@ -433,20 +433,20 @@ private void bringAllDependencies( if (includeTransitive) { bringAllDependencies( - transitiveDepList, + targetDeps, directDep, cachedTree, true, Stream.concat(path.stream(), depList.stream()).collect(Collectors.toList())); } } - transitiveDepList.sort( + targetDeps.sort( (map1, map2) -> { String string1 = (String) (map1.get("name")); String string2 = (String) (map2.get("name")); return Arrays.compare(string1.toCharArray(), string2.toCharArray()); }); - dataMap.put("dependencies", transitiveDepList); + dataMap.put("dependencies", targetDeps); } } From 421aee92a2334f35d5d0157ec5bb31f33702f510 Mon Sep 17 00:00:00 2001 From: Jude Niroshan Date: Wed, 8 May 2024 12:20:41 +0200 Subject: [PATCH 4/6] refactor: remove unused block Signed-off-by: Jude Niroshan --- .../java/com/redhat/exhort/utils/PythonControllerBase.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java b/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java index 6392dd7..fc625e5 100644 --- a/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java +++ b/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java @@ -341,7 +341,7 @@ List getDependencyTreeJsonFromPipDepTree() { executeCommandOrExtractFromEnv( "EXHORT_PIP_PIPDEPTREE", pipBinaryLocation, "install", "pipdeptree"); - String pipdeptreeJsonString; + String pipdeptreeJsonString = ""; if (isVirtualEnv()) { pipdeptreeJsonString = executeCommandOrExtractFromEnv("EXHORT_PIP_PIPDEPTREE", "./bin/pipdeptree", "--json"); @@ -349,10 +349,6 @@ List getDependencyTreeJsonFromPipDepTree() { pipdeptreeJsonString = executeCommandOrExtractFromEnv( "EXHORT_PIP_PIPDEPTREE", pathToPythonBin, "-m", "pipdeptree", "--json"); - } else { - pipdeptreeJsonString = - executeCommandOrExtractFromEnv( - "EXHORT_PIP_PIPDEPTREE", "./bin/pipdeptree", "--json", "--python", pathToPythonBin); } if (debugLoggingIsNeeded()) { String pipdeptreeMessage = From b7b2a3add733d37b3f00e877ddb5873b4bbc5948 Mon Sep 17 00:00:00 2001 From: Jude Niroshan Date: Thu, 9 May 2024 14:17:53 +0200 Subject: [PATCH 5/6] refactor: remove default constructor Signed-off-by: Jude Niroshan --- src/main/java/com/redhat/exhort/utils/PythonDependency.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/redhat/exhort/utils/PythonDependency.java b/src/main/java/com/redhat/exhort/utils/PythonDependency.java index e553b2d..c77a77f 100644 --- a/src/main/java/com/redhat/exhort/utils/PythonDependency.java +++ b/src/main/java/com/redhat/exhort/utils/PythonDependency.java @@ -22,8 +22,6 @@ public class PythonDependency { String version = ""; List dependencies; - public PythonDependency() {} - public PythonDependency(String name, String version, List dependencies) { this.name = name; this.version = version; From d34f4a5cf148faabbe8e14f5cdf412e9c76e0998 Mon Sep 17 00:00:00 2001 From: Jude Niroshan Date: Thu, 9 May 2024 14:36:31 +0200 Subject: [PATCH 6/6] refactor: provide context in runtime error Signed-off-by: Jude Niroshan --- .../java/com/redhat/exhort/utils/PythonControllerBase.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java b/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java index fc625e5..da28cdc 100644 --- a/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java +++ b/src/main/java/com/redhat/exhort/utils/PythonControllerBase.java @@ -360,7 +360,7 @@ List getDependencyTreeJsonFromPipDepTree() { return mapToPythonDependencies(pipdeptreeJsonString); } - public static List mapToPythonDependencies(String jsonString) { + List mapToPythonDependencies(String jsonString) { try { // Parse JSON and store in a list of JsonNodes List jsonNodeList = new ArrayList<>(); @@ -384,7 +384,8 @@ public static List mapToPythonDependencies(String jsonString) }) .collect(Collectors.toList()); } catch (JsonProcessingException e) { - throw new RuntimeException(e); + throw new RuntimeException( + "Could not parse the JSON output from 'pipdeptree --json' command. ", e); } }