feat: support Pipenv - Pipfile.lock

ant/pom.xml

@@ -20,7 +20,7 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>8.0.3-SNAPSHOT</version>
+        <version>8.1.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>dependency-check-ant</artifactId>

archetype/pom.xml

@@ -20,7 +20,7 @@ Copyright (c) 2017 Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>8.0.3-SNAPSHOT</version>
+        <version>8.1.0-SNAPSHOT</version>
     </parent>
     <artifactId>dependency-check-plugin</artifactId>
     <name>Dependency-Check Plugin Archetype</name>

cli/pom.xml

@@ -20,7 +20,7 @@ Copyright (c) 2012 - Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>8.0.3-SNAPSHOT</version>
+        <version>8.1.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>dependency-check-cli</artifactId>

core/pom.xml

@@ -20,7 +20,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>8.0.3-SNAPSHOT</version>
+        <version>8.1.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>dependency-check-core</artifactId>

core/src/main/java/org/owasp/dependencycheck/analyzer/PipfileAnalyzer.java

@@ -44,7 +44,11 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Used to analyze Pipfile dependency files named Pipfile.
+ * Used to analyze dependencies defined in Pipfile. This analyzer works in
+ * tandem with the `PipfilelockAnalyzer` - and both analyzers use the same key
+ * to enable/disable the analyzers. The PipfilelockAnalyzer will be used over
+ * the Pipfile if the lock file exists.
+ *
  *
  * @author fcano
  */
@@ -55,12 +59,15 @@ public class PipfileAnalyzer extends AbstractFileTypeAnalyzer {
     /**
      * The logger.
      */
-    private static final Logger LOGGER = LoggerFactory.getLogger(PythonPackageAnalyzer.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(PipfileAnalyzer.class);
     /**
      * "Pipfile" file.
      */
-    private static final String REQUIREMENTS = "Pipfile";
-
+    private static final String PIPFILE = "Pipfile";
+    /**
+     * "Pipfile.lock" file.
+     */
+    private static final String LOCKFILE = "Pipfile.lock";
     /**
      * The name of the analyzer.
      */
@@ -79,7 +86,7 @@ public class PipfileAnalyzer extends AbstractFileTypeAnalyzer {
     /**
      * The file filter used to determine which files this analyzer supports.
      */
-    private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFilenames(REQUIREMENTS).build();
+    private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFilenames(PIPFILE).build();
 
     /**
      * Returns the FileFilter
@@ -124,51 +131,54 @@ protected String getAnalyzerEnabledSettingKey() {
 
     @Override
     protected void analyzeDependency(Dependency dependency, Engine engine) throws AnalysisException {
-        LOGGER.debug("Checking file {}", dependency.getActualFilePath());
-
-        if (REQUIREMENTS.equals(dependency.getFileName())) {
-            engine.removeDependency(dependency);
+        engine.removeDependency(dependency);
+        final File lock = new File(dependency.getActualFile().getParentFile(), LOCKFILE);
+        if (lock.isFile()) {
+            LOGGER.debug("Skipping {} because a lock file was identified", dependency.getActualFilePath());
+            return;
+        } else {
+            LOGGER.debug("Checking file {}", dependency.getActualFilePath());
         }
+
         final File dependencyFile = dependency.getActualFile();
         if (!dependencyFile.isFile() || dependencyFile.length() == 0) {
             return;
         }
 
         final File actualFile = dependency.getActualFile();
-        if (actualFile.getName().equals(REQUIREMENTS)) {
-            final String contents = getFileContents(actualFile);
-            if (!contents.isEmpty()) {
-                final Matcher matcher = PACKAGE_VERSION.matcher(contents);
-                while (matcher.find()) {
-                    final String identifiedPackage = matcher.group(1);
-                    final String identifiedVersion = matcher.group(2);
-                    LOGGER.debug(String.format("package, version: %s %s", identifiedPackage, identifiedVersion));
-                    final Dependency d = new Dependency(dependency.getActualFile(), true);
-                    d.setName(identifiedPackage);
-                    d.setVersion(identifiedVersion);
-                    try {
-                        final PackageURL purl = PackageURLBuilder.aPackageURL()
-                                .withType("pypi")
-                                .withName(identifiedPackage)
-                                .withVersion(identifiedVersion)
-                                .build();
-                        d.addSoftwareIdentifier(new PurlIdentifier(purl, Confidence.HIGHEST));
-                    } catch (MalformedPackageURLException ex) {
-                        LOGGER.debug("Unable to build package url for pypi", ex);
-                        d.addSoftwareIdentifier(new GenericIdentifier("pypi:" + identifiedPackage + "@" + identifiedVersion, Confidence.HIGH));
-                    }
-                    d.setPackagePath(String.format("%s:%s", identifiedPackage, identifiedVersion));
-                    d.setEcosystem(PythonDistributionAnalyzer.DEPENDENCY_ECOSYSTEM);
-                    final String filePath = String.format("%s:%s/%s", dependency.getFilePath(), identifiedPackage, identifiedVersion);
-                    d.setFilePath(filePath);
-                    d.setSha1sum(Checksum.getSHA1Checksum(filePath));
-                    d.setSha256sum(Checksum.getSHA256Checksum(filePath));
-                    d.setMd5sum(Checksum.getMD5Checksum(filePath));
-                    d.addEvidence(EvidenceType.VENDOR, REQUIREMENTS, "vendor", identifiedPackage, Confidence.HIGHEST);
-                    d.addEvidence(EvidenceType.PRODUCT, REQUIREMENTS, "product", identifiedPackage, Confidence.HIGHEST);
-                    d.addEvidence(EvidenceType.VERSION, REQUIREMENTS, "version", identifiedVersion, Confidence.HIGHEST);
-                    engine.addDependency(d);
+
+        final String contents = getFileContents(actualFile);
+        if (!contents.isEmpty()) {
+            final Matcher matcher = PACKAGE_VERSION.matcher(contents);
+            while (matcher.find()) {
+                final String identifiedPackage = matcher.group(1);
+                final String identifiedVersion = matcher.group(2);
+                LOGGER.debug(String.format("package, version: %s %s", identifiedPackage, identifiedVersion));
+                final Dependency d = new Dependency(dependency.getActualFile(), true);
+                d.setName(identifiedPackage);
+                d.setVersion(identifiedVersion);
+                try {
+                    final PackageURL purl = PackageURLBuilder.aPackageURL()
+                            .withType("pypi")
+                            .withName(identifiedPackage)
+                            .withVersion(identifiedVersion)
+                            .build();
+                    d.addSoftwareIdentifier(new PurlIdentifier(purl, Confidence.HIGHEST));
+                } catch (MalformedPackageURLException ex) {
+                    LOGGER.debug("Unable to build package url for pypi", ex);
+                    d.addSoftwareIdentifier(new GenericIdentifier("pypi:" + identifiedPackage + "@" + identifiedVersion, Confidence.HIGH));
                 }
+                d.setPackagePath(String.format("%s:%s", identifiedPackage, identifiedVersion));
+                d.setEcosystem(PythonDistributionAnalyzer.DEPENDENCY_ECOSYSTEM);
+                final String filePath = String.format("%s:%s/%s", dependency.getFilePath(), identifiedPackage, identifiedVersion);
+                d.setFilePath(filePath);
+                d.setSha1sum(Checksum.getSHA1Checksum(filePath));
+                d.setSha256sum(Checksum.getSHA256Checksum(filePath));
+                d.setMd5sum(Checksum.getMD5Checksum(filePath));
+                d.addEvidence(EvidenceType.VENDOR, PIPFILE, "vendor", identifiedPackage, Confidence.HIGHEST);
+                d.addEvidence(EvidenceType.PRODUCT, PIPFILE, "product", identifiedPackage, Confidence.HIGHEST);
+                d.addEvidence(EvidenceType.VERSION, PIPFILE, "version", identifiedVersion, Confidence.HIGHEST);
+                engine.addDependency(d);
             }
         }
     }

core/src/main/java/org/owasp/dependencycheck/analyzer/PipfilelockAnalyzer.java

@@ -0,0 +1,219 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2023 The OWASP Foundation. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.analyzer;
+
+import org.apache.commons.io.FileUtils;
+import com.github.packageurl.MalformedPackageURLException;
+import com.github.packageurl.PackageURL;
+import com.github.packageurl.PackageURLBuilder;
+import java.io.BufferedInputStream;
+import org.owasp.dependencycheck.Engine;
+import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
+import org.owasp.dependencycheck.dependency.Confidence;
+import org.owasp.dependencycheck.dependency.Dependency;
+import org.owasp.dependencycheck.utils.FileFilterBuilder;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.dependency.EvidenceType;
+import org.owasp.dependencycheck.exception.InitializationException;
+import org.owasp.dependencycheck.dependency.naming.GenericIdentifier;
+import org.owasp.dependencycheck.dependency.naming.PurlIdentifier;
+import org.owasp.dependencycheck.utils.Checksum;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.Set;
+import javax.annotation.concurrent.ThreadSafe;
+import javax.json.Json;
+import javax.json.JsonException;
+import javax.json.JsonObject;
+import javax.json.JsonReader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Used to analyze dependencies defined in Pipfile.lock. This analyzer works in
+ * tandem with the `PipfileAnalyzer` - and both analyzers use the same key to
+ * enable/disable the analyzers. The PipfileAnalyzer will be skipped if the lock
+ * file exists, as the lock will provide more accurate version numbers.
+ *
+ * @author jeremy.long
+ */
+@Experimental
+@ThreadSafe
+public class PipfilelockAnalyzer extends AbstractFileTypeAnalyzer {
+
+    /**
+     * The logger.
+     */
+    private static final Logger LOGGER = LoggerFactory.getLogger(PipfilelockAnalyzer.class);
+    /**
+     * "Pipfile.lock" file.
+     */
+    private static final String LOCKFILE = "Pipfile.lock";
+
+    /**
+     * The identifiedPackage of the analyzer.
+     */
+    private static final String ANALYZER_NAME = "Pipfile.lock Analyzer";
+
+    /**
+     * The phase that this analyzer is intended to run in.
+     */
+    private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION;
+
+    /**
+     * The file filter used to determine which files this analyzer supports.
+     */
+    private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFilenames(LOCKFILE).build();
+
+    /**
+     * Returns the FileFilter
+     *
+     * @return the FileFilter
+     */
+    @Override
+    protected FileFilter getFileFilter() {
+        return FILTER;
+    }
+
+    /**
+     * Returns the identifiedPackage of the analyzer.
+     *
+     * @return the identifiedPackage of the analyzer.
+     */
+    @Override
+    public String getName() {
+        return ANALYZER_NAME;
+    }
+
+    /**
+     * Returns the phase that the analyzer is intended to run in.
+     *
+     * @return the phase that the analyzer is intended to run in.
+     */
+    @Override
+    public AnalysisPhase getAnalysisPhase() {
+        return ANALYSIS_PHASE;
+    }
+
+    /**
+     * Returns the key used in the properties file to reference the analyzer's
+     * enabled property.
+     *
+     * @return the analyzer's enabled property setting key
+     */
+    @Override
+    protected String getAnalyzerEnabledSettingKey() {
+        return Settings.KEYS.ANALYZER_PIPFILE_ENABLED;
+    }
+
+    @Override
+    protected void analyzeDependency(Dependency dependency, Engine engine) throws AnalysisException {
+        LOGGER.debug("Checking file {}", dependency.getActualFilePath());
+
+        engine.removeDependency(dependency);
+
+        final File dependencyFile = dependency.getActualFile();
+        if (!dependencyFile.isFile() || dependencyFile.length() == 0) {
+            return;
+        }
+
+        final File actualFile = dependency.getActualFile();
+        try (FileInputStream fin = new FileInputStream(actualFile);
+                InputStream in = new BufferedInputStream(fin);
+                JsonReader jsonReader = Json.createReader(in)) {
+
+            final JsonObject jsonLock = jsonReader.readObject();
+            final JsonObject develop = jsonLock.getJsonObject("develop");
+            final Set<String> keys = develop.keySet();
+            for (String identifiedPackage : keys) {
+                final JsonObject dep = develop.getJsonObject(identifiedPackage);
+                final String selectedVersion = dep.getString("version", "").trim();
+                if (selectedVersion.startsWith("==") && selectedVersion.length() > 2) {
+                    final String identifiedVersion = selectedVersion.substring(2).trim();
+                    LOGGER.debug("package, version: {} {}", identifiedPackage, identifiedVersion);
+
+                    final Dependency d = new Dependency(dependency.getActualFile(), true);
+                    d.setName(identifiedPackage);
+                    d.setVersion(identifiedVersion);
+                    try {
+                        final PackageURL purl = PackageURLBuilder.aPackageURL()
+                                .withType("pypi")
+                                .withName(identifiedPackage)
+                                .withVersion(identifiedVersion)
+                                .build();
+                        d.addSoftwareIdentifier(new PurlIdentifier(purl, Confidence.HIGHEST));
+                    } catch (MalformedPackageURLException ex) {
+                        LOGGER.debug("Unable to build package url for pypi", ex);
+                        d.addSoftwareIdentifier(new GenericIdentifier("pypi:" + identifiedPackage + "@" + identifiedVersion, Confidence.HIGH));
+                    }
+                    d.setPackagePath(String.format("%s:%s", identifiedPackage, identifiedVersion));
+                    d.setEcosystem(PythonDistributionAnalyzer.DEPENDENCY_ECOSYSTEM);
+                    final String filePath = String.format("%s:%s/%s", dependency.getFilePath(), identifiedPackage, identifiedVersion);
+                    d.setFilePath(filePath);
+                    d.setSha1sum(Checksum.getSHA1Checksum(filePath));
+                    d.setSha256sum(Checksum.getSHA256Checksum(filePath));
+                    d.setMd5sum(Checksum.getMD5Checksum(filePath));
+                    d.addEvidence(EvidenceType.VENDOR, LOCKFILE, "vendor", identifiedPackage, Confidence.HIGHEST);
+                    d.addEvidence(EvidenceType.PRODUCT, LOCKFILE, "product", identifiedPackage, Confidence.HIGHEST);
+                    d.addEvidence(EvidenceType.VERSION, LOCKFILE, "version", identifiedVersion, Confidence.HIGHEST);
+                    engine.addDependency(d);
+                } else {
+                    LOGGER.debug("Skipping `{}`: Unknown version `{}` in `{}`", identifiedPackage, selectedVersion, dependency.getActualFilePath());
+                }
+            }
+        } catch (FileNotFoundException ex) {
+            throw new RuntimeException(ex);
+        } catch (IOException | JsonException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    /**
+     * Retrieves the contents of a given file.
+     *
+     * @param actualFile the file to read
+     * @return the contents of the file
+     * @throws AnalysisException thrown if there is an IO Exception
+     */
+    private String getFileContents(final File actualFile) throws AnalysisException {
+        try {
+            return FileUtils.readFileToString(actualFile, Charset.defaultCharset()).trim();
+        } catch (IOException e) {
+            throw new AnalysisException("Problem occurred while reading dependency file.", e);
+        }
+    }
+
+    /**
+     * Initializes the file type analyzer.
+     *
+     * @param engine a reference to the dependency-check engine
+     * @throws InitializationException thrown if there is an exception during
+     * initialization
+     */
+    @Override
+    protected void prepareFileTypeAnalyzer(Engine engine) throws InitializationException {
+        // No initialization needed.
+    }
+}

core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer

@@ -21,6 +21,7 @@ org.owasp.dependencycheck.analyzer.PythonDistributionAnalyzer
 org.owasp.dependencycheck.analyzer.PythonPackageAnalyzer
 org.owasp.dependencycheck.analyzer.PipAnalyzer
 org.owasp.dependencycheck.analyzer.PipfileAnalyzer
+org.owasp.dependencycheck.analyzer.PipfilelockAnalyzer
 org.owasp.dependencycheck.analyzer.PoetryAnalyzer
 org.owasp.dependencycheck.analyzer.AutoconfAnalyzer
 org.owasp.dependencycheck.analyzer.OpenSSLAnalyzer