diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java index 7ef1560497b..365eb044b85 100644 --- a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java +++ b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java @@ -100,6 +100,10 @@ public class Update extends Purge { * The number of hours to wait before re-checking for updates. */ private Integer cveValidForHours; + /** + * Specify the first year of NVD CVE data to download; default is 2002. + */ + private Integer cveStartYear; /** * Construct a new UpdateTask. @@ -381,6 +385,29 @@ public void setCveValidForHours(Integer cveValidForHours) { this.cveValidForHours = cveValidForHours; } + /** + * Get the value of cveStartYear. + * + * @return the value of cveStartYear + */ + public Integer getCveStartYear() { + return cveStartYear; + } + + /** + * Set the value of cveStartYear. + * + * @param cveStartYear new value of cveStartYear + */ + public void setCveStartYear(Integer cveStartYear) { + if (cveStartYear != null && cveStartYear < 2002) { + log("Invalid Configuration: cveStartYear must be 2002 or greater", Project.MSG_ERR); + this.cveStartYear = 2002; + } else { + this.cveStartYear = cveStartYear; + } + } + /** * Executes the update by initializing the settings, downloads the NVD XML * data, and then processes the data storing it in the local database. @@ -441,6 +468,7 @@ protected void populateSettings() throws BuildException { getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON, cveModifiedJson); getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON, cveUrlBase); getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, cveWaitTime); + getSettings().setIntIfNotNull(Settings.KEYS.CVE_START_YEAR, cveStartYear); if (cveValidForHours != null) { if (cveValidForHours >= 0) { getSettings().setInt(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, cveValidForHours); diff --git a/ant/src/site/markdown/config-update.md b/ant/src/site/markdown/config-update.md index 2ce489f320b..7fc9d606d77 100644 --- a/ant/src/site/markdown/config-update.md +++ b/ant/src/site/markdown/config-update.md @@ -37,6 +37,7 @@ Property | Description cveUrlModified | URL for the modified CVE JSON data feed. When mirroring the NVD you must mirror the *.json.gz and the *.meta files. Optional if your custom cveUrlBase is just a domain name change. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz cveUrlBase | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz cveWaitTime | The time in milliseconds to wait between downloads from the NVD. | 4000 +cveStartYear | The first year of NVD CVE data to download from the NVD. | 2002 dataDirectory | Data directory that is used to store the local copy of the NVD. This should generally not be changed. | data databaseDriverName | The name of the database driver. Example: org.h2.Driver. |   databaseDriverPath | The path to the database driver JAR file; only used if the driver is not in the class path. |   diff --git a/ant/src/site/markdown/configuration.md b/ant/src/site/markdown/configuration.md index 98a63fc40d3..c4e990e7273 100644 --- a/ant/src/site/markdown/configuration.md +++ b/ant/src/site/markdown/configuration.md @@ -137,13 +137,13 @@ may be the cvedUrl properties, which can be used to host a mirror of the NVD wit Property | Description | Default Value ---------------------|--------------------------------------------------------------------------|------------------ -cveUrl12Modified | URL for the modified CVE 1.2. | http://nvd.nist.gov/download/nvdcve-modified.xml -cveUrl20Modified | URL for the modified CVE 2.0. | http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml -cveUrl12Base | Base URL for each year's CVE 1.2, the %d will be replaced with the year. | http://nvd.nist.gov/download/nvdcve-%d.xml -cveUrl20Base | Base URL for each year's CVE 2.0, the %d will be replaced with the year. | http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml -dataDirectory | Data directory that is used to store the local copy of the NVD. This should generally not be changed. | data -databaseDriverName | The name of the database driver. Example: org.h2.Driver. |   -databaseDriverPath | The path to the database driver JAR file; only used if the driver is not in the class path. |   +cveUrlModified | URL for the modified CVE JSON data feed. When mirroring the NVD you must mirror the *.json.gz and the *.meta files. Optional if your custom cveUrlBase is just a domain name change. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz +cveUrlBase | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz +cveWaitTime | The time in milliseconds to wait between downloads from the NVD. | 4000 +cveStartYear | The first year of NVD CVE data to download from the NVD. | 2002 +dataDirectory | Data directory that is used to store the local copy of the NVD. This should generally not be changed. | data +databaseDriverName | The name of the database driver. Example: org.h2.Driver. |   +databaseDriverPath | The path to the database driver JAR file; only used if the driver is not in the class path. |   connectionString | The connection string used to connect to the database. See using a [database server](../data/database.html). |   -databaseUser | The username used when connecting to the database. |   -databasePassword | The password used when connecting to the database. |   +databaseUser | The username used when connecting to the database. |   +databasePassword | The password used when connecting to the database. |   diff --git a/cli/src/main/java/org/owasp/dependencycheck/App.java b/cli/src/main/java/org/owasp/dependencycheck/App.java index 433b0540c99..a9670722552 100644 --- a/cli/src/main/java/org/owasp/dependencycheck/App.java +++ b/cli/src/main/java/org/owasp/dependencycheck/App.java @@ -443,6 +443,8 @@ protected void populateSettings(CliParser cli) throws InvalidSettingException { cli.getStringArgument(CliParser.ARGUMENT.HINTS_FILE)); settings.setIntIfNotNull(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, cli.getIntegerValue(CliParser.ARGUMENT.CVE_VALID_FOR_HOURS)); + settings.setIntIfNotNull(Settings.KEYS.CVE_START_YEAR, + cli.getIntegerValue(CliParser.ARGUMENT.CVE_START_YEAR)); settings.setArrayIfNotEmpty(Settings.KEYS.SUPPRESSION_FILE, cli.getStringArguments(CliParser.ARGUMENT.SUPPRESSION_FILES)); //File Type Analyzer Settings diff --git a/cli/src/main/java/org/owasp/dependencycheck/CliParser.java b/cli/src/main/java/org/owasp/dependencycheck/CliParser.java index 03eab0f6492..4afa93da493 100644 --- a/cli/src/main/java/org/owasp/dependencycheck/CliParser.java +++ b/cli/src/main/java/org/owasp/dependencycheck/CliParser.java @@ -114,7 +114,7 @@ private CommandLine parseArgs(String[] args) throws ParseException { */ private void validateArgs() throws FileNotFoundException, ParseException { if (isUpdateOnly() || isRunScan()) { - final String value = line.getOptionValue(ARGUMENT.CVE_VALID_FOR_HOURS); + String value = line.getOptionValue(ARGUMENT.CVE_VALID_FOR_HOURS); if (value != null) { try { final int i = Integer.parseInt(value); @@ -125,6 +125,18 @@ private void validateArgs() throws FileNotFoundException, ParseException { throw new ParseException("Invalid Setting: cveValidForHours must be a number greater than or equal to 0."); } } + value = line.getOptionValue(ARGUMENT.CVE_START_YEAR); + if (value != null) { + try { + final int i = Integer.parseInt(value); + if (i < 2002) { + throw new ParseException("Invalid Setting: cveStartYear must be a number greater than or equal to 2002."); + } + } catch (NumberFormatException ex) { + throw new ParseException("Invalid Setting: cveStartYear must be a number greater than or equal to 2002."); + } + } + } if (isRunScan()) { validatePathExists(getScanFiles(), ARGUMENT.SCAN); @@ -376,6 +388,8 @@ private void addAdvancedOptions(final Options options) { "The path to the `yarn` executable.")) .addOption(newOptionWithArg(ARGUMENT.CVE_VALID_FOR_HOURS, "hours", "The number of hours to wait before checking for new updates from the NVD.")) + .addOption(newOptionWithArg(ARGUMENT.CVE_START_YEAR, "year", + "The first year to retrieve NVD CVE data for; default is 2002.")) .addOption(newOptionWithArg(ARGUMENT.RETIREJS_FILTERS, "pattern", "Specify Retire JS content filter used to exclude files from analysis based on their content; " + "most commonly used to exclude based on your applications own copyright line. This " @@ -1085,6 +1099,11 @@ public static class ARGUMENT { * checking for new updates from the NVD. */ public static final String CVE_VALID_FOR_HOURS = "cveValidForHours"; + /** + * The CLI argument name for setting the first year to retrieve NVD + * data. + */ + public static final String CVE_START_YEAR = "cveStartYear"; /** * The username for basic auth to the CVE data. */ @@ -1231,7 +1250,7 @@ public static class ARGUMENT { */ public static final String DISABLE_NODE_JS = "disableNodeJS"; /** - * Skips dev dependencies in Node Package Analyzer + * Skips dev dependencies in Node Package Analyzer. */ public static final String NODE_PACKAGE_SKIP_DEV_DEPENDENCIES = "nodePackageSkipDevDependencies"; /** diff --git a/cli/src/main/resources/completion-for-dependency-check.sh b/cli/src/main/resources/completion-for-dependency-check.sh index 65ed62bc207..b6e1e1584bb 100755 --- a/cli/src/main/resources/completion-for-dependency-check.sh +++ b/cli/src/main/resources/completion-for-dependency-check.sh @@ -21,7 +21,8 @@ _odc_completions() --connectionString --cveUrlBase --cveUrlModified - --cveValidForHours + --cveValidForHours + --cveStartYear --cveUser --cvePassword --cveDownloadWait diff --git a/core/src/main/java/org/owasp/dependencycheck/Engine.java b/core/src/main/java/org/owasp/dependencycheck/Engine.java index 0906d3b24d2..d950b86286a 100644 --- a/core/src/main/java/org/owasp/dependencycheck/Engine.java +++ b/core/src/main/java/org/owasp/dependencycheck/Engine.java @@ -891,7 +891,7 @@ public boolean doUpdates(boolean remainOpen) throws UpdateException, DatabaseExc //lock is not needed as we already have the lock held openDatabase(true, false); } - + return dbUpdatesMade; } catch (WriteLockException ex) { throw new UpdateException("Unable to obtain an exclusive lock on the H2 database to perform updates", ex); diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java index 41ca92ff660..0aaf21e7ac5 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java @@ -235,7 +235,9 @@ private void processPom(Dependency dependency, MavenArtifact ma) throws IOExcept LOGGER.debug("Downloading {}", ma.getPomUrl()); //TODO add caching final Downloader downloader = new Downloader(getSettings()); - downloader.fetchFile(new URL(ma.getPomUrl()), pomFile, Settings.KEYS.ANALYZER_ARTIFACTORY_API_USERNAME, Settings.KEYS.ANALYZER_ARTIFACTORY_API_TOKEN); + downloader.fetchFile(new URL(ma.getPomUrl()), pomFile, + Settings.KEYS.ANALYZER_ARTIFACTORY_API_USERNAME, + Settings.KEYS.ANALYZER_ARTIFACTORY_API_TOKEN); PomUtils.analyzePOM(dependency, pomFile); } catch (DownloadFailedException ex) { diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/NodePackageAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/NodePackageAnalyzer.java index 1d91cd5c410..a1789f1b0a5 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/NodePackageAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/NodePackageAnalyzer.java @@ -334,7 +334,7 @@ private void processDependencies(JsonObject json, File baseDir, File rootFile, String parentPackage, Engine engine) throws AnalysisException { if (json.containsKey("dependencies")) { final JsonObject deps = json.getJsonObject("dependencies"); - boolean skipDev = getSettings().getBoolean(Settings.KEYS.ANALYZER_NODE_PACKAGE_SKIPDEV, false); + final boolean skipDev = getSettings().getBoolean(Settings.KEYS.ANALYZER_NODE_PACKAGE_SKIPDEV, false); for (Map.Entry entry : deps.entrySet()) { final String name = entry.getKey(); final String version; diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/OssIndexAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/OssIndexAnalyzer.java index 9ff8942278e..3e73a6b5848 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/OssIndexAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/OssIndexAnalyzer.java @@ -17,7 +17,6 @@ */ package org.owasp.dependencycheck.analyzer; -import java.net.URI; import org.sonatype.ossindex.service.api.componentreport.ComponentReport; import org.sonatype.ossindex.service.api.componentreport.ComponentReportVulnerability; import org.sonatype.ossindex.service.api.cvss.Cvss2Severity; @@ -161,11 +160,12 @@ protected void analyzeDependency(final Dependency dependency, final Engine engin } /** - * Delays each request (thread) by the configured amount of seconds, if the configuration is present. + * Delays each request (thread) by the configured amount of seconds, if the + * configuration is present. */ private void requestDelay() throws InterruptedException { - final int delay = getSettings().getInt(Settings.KEYS.ANALYZER_OSSINDEX_REQUEST_DELAY,0); - if(delay > 0) { + final int delay = getSettings().getInt(Settings.KEYS.ANALYZER_OSSINDEX_REQUEST_DELAY, 0); + if (delay > 0) { LOG.debug("Request delay: " + delay); TimeUnit.SECONDS.sleep(delay); } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index f6825715045..268bd17efe4 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -302,8 +302,9 @@ public boolean isOpen() { * @throws DatabaseException throw if there is an error generating the * prepared statement */ - private PreparedStatement getPreparedStatement(Connection connection, PreparedStatementCveDb key, String parameter) throws DatabaseException, SQLException { - PreparedStatement preparedStatement = getPreparedStatement(connection, key); + private PreparedStatement getPreparedStatement(Connection connection, PreparedStatementCveDb key, String parameter) + throws DatabaseException, SQLException { + final PreparedStatement preparedStatement = getPreparedStatement(connection, key); preparedStatement.setString(1, parameter); return preparedStatement; } @@ -320,8 +321,9 @@ private PreparedStatement getPreparedStatement(Connection connection, PreparedSt * @throws DatabaseException throw if there is an error generating the * prepared statement */ - private PreparedStatement getPreparedStatement(Connection connection, PreparedStatementCveDb key, int parameter) throws DatabaseException, SQLException { - PreparedStatement preparedStatement = getPreparedStatement(connection, key); + private PreparedStatement getPreparedStatement(Connection connection, PreparedStatementCveDb key, int parameter) + throws DatabaseException, SQLException { + final PreparedStatement preparedStatement = getPreparedStatement(connection, key); preparedStatement.setInt(1, parameter); return preparedStatement; } @@ -410,9 +412,6 @@ public Set getCPEs(String vendor, String product) { final Set cpe = new HashSet<>(); try (Connection conn = databaseManager.getConnection(); PreparedStatement ps = getPreparedStatement(conn, SELECT_CPE_ENTRIES)) { - if (ps == null) { - throw new SQLException("Database query does not exist in the resource bundle: " + SELECT_CPE_ENTRIES); - } //part, vendor, product, version, update_version, edition, //lang, sw_edition, target_sw, target_hw, other, ecosystem ps.setString(1, vendor); @@ -526,16 +525,10 @@ public void saveProperty(String key, String value) { } else { // No Merge statement, so doing an Update/Insert... try (PreparedStatement updateProperty = getPreparedStatement(conn, UPDATE_PROPERTY)) { - if (updateProperty == null) { - throw new SQLException("Database query does not exist in the resource bundle: " + UPDATE_PROPERTY); - } updateProperty.setString(1, value); updateProperty.setString(2, key); if (updateProperty.executeUpdate() == 0) { try (PreparedStatement insertProperty = getPreparedStatement(conn, INSERT_PROPERTY)) { - if (insertProperty == null) { - throw new SQLException("Database query does not exist in the resource bundle: " + INSERT_PROPERTY); - } insertProperty.setString(1, key); insertProperty.setString(2, value); insertProperty.executeUpdate(); @@ -852,9 +845,6 @@ private int updateOrInsertVulnerability(DefCveItem cve, String description) { final int vulnerabilityId; try (Connection conn = databaseManager.getConnection(); PreparedStatement callUpdate = getPreparedStatement(conn, UPDATE_VULNERABILITY)) { - if (callUpdate == null) { - throw new SQLException("Database query does not exist in the resource bundle: " + UPDATE_VULNERABILITY); - } // String 1.cve, String 2.description, String 3.v2Severity, Float 4.v2ExploitabilityScore, // Float 5.v2ImpactScore, Boolean 6.v2AcInsufInfo, Boolean 7.v2ObtainAllPrivilege, // Boolean 8.v2ObtainUserPrivilege, Boolean 9.v2ObtainOtherPrivilege, Boolean 10.v2UserInteractionRequired, @@ -974,10 +964,6 @@ private int updateOrInsertVulnerability(DefCveItem cve, String description) { private void updateVulnerabilityInsertCwe(int vulnerabilityId, DefCveItem cve) throws SQLException { try (Connection conn = databaseManager.getConnection(); PreparedStatement insertCWE = getPreparedStatement(conn, INSERT_CWE, vulnerabilityId)) { - if (insertCWE == null) { - throw new SQLException("Database query does not exist in the resource bundle: " + INSERT_CWE); - } - for (ProblemtypeDatum datum : cve.getCve().getProblemtype().getProblemtypeData()) { for (LangString desc : datum.getDescription()) { if ("en".equals(desc.getLang())) { @@ -1028,9 +1014,6 @@ private void updateVulnerabilityInsertSoftware(int vulnerabilityId, String cveId throws DatabaseException, SQLException { try (Connection conn = databaseManager.getConnection(); PreparedStatement insertSoftware = getPreparedStatement(conn, INSERT_SOFTWARE)) { - if (insertSoftware == null) { - throw new SQLException("Database query does not exist in the resource bundle: " + INSERT_SOFTWARE); - } for (VulnerableSoftware parsedCpe : software) { insertSoftware.setInt(1, vulnerabilityId); insertSoftware.setString(2, parsedCpe.getPart().getAbbreviation()); @@ -1086,9 +1069,6 @@ private void updateVulnerabilityInsertSoftware(int vulnerabilityId, String cveId private void updateVulnerabilityInsertReferences(int vulnerabilityId, DefCveItem cve) throws SQLException { try (Connection conn = databaseManager.getConnection(); PreparedStatement insertReference = getPreparedStatement(conn, INSERT_REFERENCE)) { - if (insertReference == null) { - throw new SQLException("Database query does not exist in the resource bundle: " + INSERT_REFERENCE); - } if (cve.getCve().getReferences() != null) { for (Reference r : cve.getCve().getReferences().getReferenceData()) { insertReference.setInt(1, vulnerabilityId); @@ -1351,62 +1331,6 @@ protected VulnerableSoftware getMatchingSoftware(Cpe cpe, Set majorVersionsAffectingAllPrevious = new HashSet<>(); -// final boolean matchesAnyPrevious = identifiedVersion == null || "-".equals(identifiedVersion.toString()); -// String majorVersionMatch = null; -// for (Entry entry : vulnerableSoftware.entrySet()) { -// final DependencyVersion v = parseDependencyVersion(entry.getKey()); -// if (v == null || "-".equals(v.toString())) { //all versions -// return entry; -// } -// if (entry.getValue()) { -// if (matchesAnyPrevious) { -// return entry; -// } -// if (identifiedVersion != null && identifiedVersion.getVersionParts().get(0).equals(v.getVersionParts().get(0))) { -// majorVersionMatch = v.getVersionParts().get(0); -// } -// majorVersionsAffectingAllPrevious.add(v.getVersionParts().get(0)); -// } -// } -// if (matchesAnyPrevious) { -// return null; -// } -// -// final boolean canSkipVersions = majorVersionMatch != null && majorVersionsAffectingAllPrevious.size() > 1; -// //yes, we are iterating over this twice. The first time we are skipping versions those that affect all versions -// //then later we process those that affect all versions. This could be done with sorting... -// for (Entry entry : vulnerableSoftware.entrySet()) { -// if (!entry.getValue()) { -// final DependencyVersion v = parseDependencyVersion(entry.getKey()); -// //this can't dereference a null 'majorVersionMatch' as canSkipVersions accounts for this. -// if (canSkipVersions && majorVersionMatch != null && !majorVersionMatch.equals(v.getVersionParts().get(0))) { -// continue; -// } -// //this can't dereference a null 'identifiedVersion' because if it was null we would have exited -// //in the above loop or just after loop (if matchesAnyPrevious return null). -// if (identifiedVersion != null && identifiedVersion.equals(v)) { -// return entry; -// } -// } -// } -// for (Entry entry : vulnerableSoftware.entrySet()) { -// if (entry.getValue()) { -// final DependencyVersion v = parseDependencyVersion(entry.getKey()); -// //this can't dereference a null 'majorVersionMatch' as canSkipVersions accounts for this. -// if (canSkipVersions && majorVersionMatch != null && !majorVersionMatch.equals(v.getVersionParts().get(0))) { -// continue; -// } -// //this can't dereference a null 'identifiedVersion' because if it was null we would have exited -// //in the above loop or just after loop (if matchesAnyPrevious return null). -// if (entry.getValue() && identifiedVersion != null && identifiedVersion.compareTo(v) <= 0 -// && !(isVersionTwoADifferentProduct && !identifiedVersion.getVersionParts().get(0).equals(v.getVersionParts().get(0)))) { -// return entry; -// } -// } -// } -// return null; } /** diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java index 23e5159c436..579c3dac8a7 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java @@ -17,20 +17,29 @@ */ package org.owasp.dependencycheck.data.update; +import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.net.MalformedURLException; import java.util.Calendar; import java.util.HashSet; import java.util.Set; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.stream.Collectors; import javax.annotation.concurrent.ThreadSafe; +import org.apache.commons.io.FileUtils; import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.data.nvd.json.MetaProperties; @@ -43,6 +52,7 @@ import org.owasp.dependencycheck.data.update.exception.InvalidDataException; import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.owasp.dependencycheck.data.update.nvd.DownloadTask; +import org.owasp.dependencycheck.data.update.nvd.NvdCache; import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo; import org.owasp.dependencycheck.data.update.nvd.ProcessTask; import org.owasp.dependencycheck.utils.DateUtil; @@ -336,11 +346,31 @@ private void performUpdate(List updateable) throws UpdateException { */ protected final MetaProperties getMetaFile(String url) throws UpdateException { final String metaUrl = url.substring(0, url.length() - 7) + "meta"; + final NvdCache cache = new NvdCache(settings); try { final URL u = new URL(metaUrl); - final Downloader d = new Downloader(settings); - final String content = d.fetchContent(u, true, Settings.KEYS.CVE_USER, Settings.KEYS.CVE_PASSWORD); - return new MetaProperties(content); + final File tmp = settings.getTempFile("nvd", "meta"); + if (cache.notInCache(u, tmp)) { + final Downloader d = new Downloader(settings); + final String content = d.fetchContent(u, true, Settings.KEYS.CVE_USER, Settings.KEYS.CVE_PASSWORD); + try (FileOutputStream fos = new FileOutputStream(tmp); + OutputStreamWriter osw = new OutputStreamWriter(fos, StandardCharsets.UTF_8); + BufferedWriter writer = new BufferedWriter(osw)) { + writer.write(content); + } + cache.storeInCache(u, tmp); + FileUtils.deleteQuietly(tmp); + return new MetaProperties(content); + } else { + final String content; + try (FileInputStream fis = new FileInputStream(tmp); + InputStreamReader isr = new InputStreamReader(fis, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(isr)) { + content = reader.lines().collect(Collectors.joining("\n")); + } + FileUtils.deleteQuietly(tmp); + return new MetaProperties(content); + } } catch (MalformedURLException ex) { throw new UpdateException("Meta file url is invalid: " + metaUrl, ex); } catch (InvalidDataException ex) { @@ -351,6 +381,8 @@ protected final MetaProperties getMetaFile(String url) throws UpdateException { throw new UpdateException("Unable to download meta file: " + metaUrl + "; received 429 -- too many requests", ex); } catch (ResourceNotFoundException ex) { throw new UpdateException("Unable to download meta file: " + metaUrl + "; received 404 -- resource not found", ex); + } catch (IOException ex) { + throw new RuntimeException(ex); } } @@ -389,18 +421,25 @@ protected final List getUpdatesNeeded() throws UpdateException { if (!needsFullUpdate && lastUpdated == modified.getLastModifiedDate()) { return updates; } else { + final int start = settings.getInt(Settings.KEYS.CVE_START_YEAR); + final int end = Calendar.getInstance().get(Calendar.YEAR); + final String baseUrl = settings.getString(Settings.KEYS.CVE_BASE_JSON); final NvdCveInfo item = new NvdCveInfo(MODIFIED, url, modified.getLastModifiedDate()); updates.add(item); - if (needsFullUpdate || !DateUtil.withinDateRange(lastUpdated, now, days)) { - final int start = settings.getInt(Settings.KEYS.CVE_START_YEAR); - final int end = Calendar.getInstance().get(Calendar.YEAR); - final String baseUrl = settings.getString(Settings.KEYS.CVE_BASE_JSON); + if (needsFullUpdate) { + // no need to download each one, just use the modified timestamp + for (int i = start; i <= end; i++) { + url = String.format(baseUrl, i); + final NvdCveInfo entry = new NvdCveInfo(Integer.toString(i), url, modified.getLastModifiedDate()); + updates.add(entry); + } + } else if (!DateUtil.withinDateRange(lastUpdated, now, days)) { final long waitTime = settings.getInt(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, 4000); for (int i = start; i <= end; i++) { try { url = String.format(baseUrl, i); - final MetaProperties meta = getMetaFile(url); Thread.sleep(waitTime); + final MetaProperties meta = getMetaFile(url); final long currentTimestamp = getPropertyInSeconds(DatabaseProperties.LAST_UPDATED_BASE + i); if (currentTimestamp < meta.getLastModifiedDate()) { diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java index 94e4d342ccf..2b0e1d0316b 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java @@ -29,7 +29,9 @@ import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.owasp.dependencycheck.utils.DownloadFailedException; import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.ResourceNotFoundException; import org.owasp.dependencycheck.utils.Settings; +import org.owasp.dependencycheck.utils.TooManyRequestsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -119,32 +121,34 @@ public File getFile() { @Override public Future call() throws Exception { + final long waitTime = settings.getInt(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, 4000); + long startDownload = 0; + final NvdCache cache = new NvdCache(settings); try { final URL url1 = new URL(nvdCveInfo.getUrl()); - LOGGER.info("Download Started for NVD CVE - {}", nvdCveInfo.getId()); - final long startDownload = System.currentTimeMillis(); - try { - final Downloader downloader = new Downloader(settings); - downloader.fetchFile(url1, file, Settings.KEYS.CVE_USER, Settings.KEYS.CVE_PASSWORD); - } catch (DownloadFailedException ex) { - LOGGER.error("Download Failed for NVD CVE - {}\nSome CVEs may not be reported. Reason: {}", - nvdCveInfo.getId(), ex.getMessage()); - if (settings.getString(Settings.KEYS.PROXY_SERVER) == null) { - LOGGER.error("If you are behind a proxy you may need to configure dependency-check to use the proxy."); + if (cache.notInCache(url1, file)) { + Thread.sleep(waitTime); + LOGGER.info("Download Started for NVD CVE - {}", nvdCveInfo.getId()); + startDownload = System.currentTimeMillis(); + //going from 2 to 4 to have 3 download attempts with easier info messages + final int downloadAttempts = 4; + for (int x = 2; x <= downloadAttempts && !attemptDownload(url1, x == downloadAttempts); x++) { + LOGGER.info("Download Attemp {} for NVD CVE - {}", x, nvdCveInfo.getId()); + Thread.sleep(waitTime * (x / 2)); + } + if (file.isFile() && file.length() > 0) { + LOGGER.info("Download Complete for NVD CVE - {} ({} ms)", nvdCveInfo.getId(), + System.currentTimeMillis() - startDownload); + cache.storeInCache(url1, file); + } else { + throw new DownloadFailedException("Unable to download NVD CVE " + nvdCveInfo.getId()); } - LOGGER.debug("", ex); - return null; } - - LOGGER.info("Download Complete for NVD CVE - {} ({} ms)", nvdCveInfo.getId(), - System.currentTimeMillis() - startDownload); if (this.processorService == null) { return null; } final ProcessTask task = new ProcessTask(cveDB, this, settings); final Future val = this.processorService.submit(task); - final long waitTime = settings.getInt(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, 4000); - Thread.sleep(waitTime); return val; } catch (Throwable ex) { @@ -157,6 +161,24 @@ public Future call() throws Exception { return null; } + private boolean attemptDownload(final URL url1, boolean showLog) throws TooManyRequestsException, ResourceNotFoundException { + try { + final Downloader downloader = new Downloader(settings); + downloader.fetchFile(url1, file, Settings.KEYS.CVE_USER, Settings.KEYS.CVE_PASSWORD); + } catch (DownloadFailedException ex) { + if (showLog) { + LOGGER.error("Download Failed for NVD CVE - {}\nSome CVEs may not be reported. Reason: {}", + nvdCveInfo.getId(), ex.getMessage()); + if (settings.getString(Settings.KEYS.PROXY_SERVER) == null) { + LOGGER.error("If you are behind a proxy you may need to configure dependency-check to use the proxy."); + } + LOGGER.debug("", ex); + } + return false; + } + return true; + } + /** * Attempts to delete the files that were downloaded. */ diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCache.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCache.java new file mode 100644 index 00000000000..1eb6c6dcdd5 --- /dev/null +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCache.java @@ -0,0 +1,109 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2021 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update.nvd; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.time.Instant; +import org.apache.commons.io.FileUtils; +import org.owasp.dependencycheck.utils.Settings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Simple four hour cache for files. + * + * @author Jeremy Long + */ +public class NvdCache { + + /** + * The Logger. + */ + private static final Logger LOGGER = LoggerFactory.getLogger(DownloadTask.class); + /** + * The settings. + */ + private final Settings settings; + + /** + * Creates a new cache for the NVD files. + * + * @param settings ODC settings + */ + public NvdCache(Settings settings) { + this.settings = settings; + } + + /** + * Checks if the file is in the cache and within four hours. If found and + * viable, the cached data will be copied to the given file. + * + * @param url the URL of the file cached + * @param file the path of the file to restore from the cache + * @return true if the URL file is not in the cache; otherwise + * false + */ + public boolean notInCache(URL url, File file) { + try { + //valid for up to four hours. + final long validEpoch = Instant.now().toEpochMilli() - 14400000; + final File tmp = new File(url.getPath()); + final String filename = tmp.getName(); + final File cache = new File(settings.getDataDirectory(), "nvdcache"); + if (!cache.isDirectory()) { + return true; + } + final File nvdFile = new File(cache, filename); + if (nvdFile.isFile() && nvdFile.lastModified() > validEpoch) { + LOGGER.debug("Copying {} from cache", url.toString()); + FileUtils.copyFile(nvdFile, file); + return false; + } + return true; + } catch (IOException ex) { + LOGGER.debug("Error checking for nvd file in cache", ex); + return true; + } + } + + /** + * Stores a file in the cache. + * + * @param url the URL of the file to cache + * @param file the file to cache + */ + public void storeInCache(URL url, File file) { + if (file.isFile()) { + try { + final File tmp = new File(url.getPath()); + final String filename = tmp.getName(); + final File cache = new File(settings.getDataDirectory(), "nvdcache"); + if (!cache.isDirectory() && !cache.mkdir()) { + return; + } + final File nvdFile = new File(cache, filename); + FileUtils.copyFile(file, nvdFile); + nvdFile.setLastModified(Instant.now().toEpochMilli()); + } catch (IOException ex) { + LOGGER.debug("Error storing nvd file in cache", ex); + } + } + } +} diff --git a/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java b/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java index 9d2f92ace45..2c6b515863d 100644 --- a/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java +++ b/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java @@ -174,10 +174,10 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma private ArtifactResolver artifactResolver; /** * The entry point towards a Maven version independent way of resolving - * dependencies (handles both Maven 3.0 Sonatype and Maven 3.1+ eclipse Aether - * implementations). Contrary to the ArtifactResolver this resolver also takes into - * account the additional repositories defined in the dependency-path towards transitive - * dependencies. + * dependencies (handles both Maven 3.0 Sonatype and Maven 3.1+ eclipse + * Aether implementations). Contrary to the ArtifactResolver this resolver + * also takes into account the additional repositories defined in the + * dependency-path towards transitive dependencies. */ @SuppressWarnings("CanBeFinal") @Component @@ -858,7 +858,6 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma @Parameter(property = "cveServerId") private String cveServerId; /** - * /** * Optionally skip excessive CVE update checks for a designated duration in * hours. */ @@ -866,6 +865,13 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma @Parameter(property = "cveValidForHours") private Integer cveValidForHours; + /** + * Specify the first year of NVD CVE data to download; default is 2002. + */ + @SuppressWarnings("CanBeFinal") + @Parameter(property = "cveStartYear") + private Integer cveStartYear; + /** * The path to dotnet core. */ @@ -1267,6 +1273,7 @@ private ExceptionCollection collectDependencyManagementDependencies(Engine engin * @return a collection of exceptions that may have occurred while resolving * and scanning the dependencies */ + //CSOFF: OperatorWrap private ExceptionCollection collectMavenDependencies(Engine engine, MavenProject project, List nodes, ProjectBuildingRequest buildingRequest, boolean aggregate) { @@ -1327,30 +1334,30 @@ private ExceptionCollection collectMavenDependencies(Engine engine, MavenProject final ArtifactCoordinate coordinate = TransferUtils.toArtifactCoordinate(dependencyNode.getArtifact()); try { final List dependencies = project.getDependencies(); - final List managedDependencies = - project.getDependencyManagement() == null ? null : project.getDependencyManagement().getDependencies(); + final List managedDependencies + = project.getDependencyManagement() == null ? null : project.getDependencyManagement().getDependencies(); final ArtifactCoordinate theCoord = TransferUtils.toArtifactCoordinate(dependencyNode.getArtifact()); if (theCoord.getClassifier() != null) { // This would trigger NPE when using the filter - MSHARED-998 getLog().debug("Expensive lookup as workaround for MSHARED-998 for " + theCoord); - final Iterable allDeps = - dependencyResolver.resolveDependencies(buildingRequest, dependencies, managedDependencies, - null); + final Iterable allDeps + = dependencyResolver.resolveDependencies(buildingRequest, dependencies, managedDependencies, + null); result = findClassifierArtifactInAllDeps(allDeps, theCoord); } else { final TransformableFilter filter = new PatternInclusionsFilter( Collections.singletonList( TransferUtils.toArtifactCoordinate(dependencyNode.getArtifact()).toString())); - final Iterable singleResult = - dependencyResolver.resolveDependencies(buildingRequest, dependencies, managedDependencies, - filter); + final Iterable singleResult + = dependencyResolver.resolveDependencies(buildingRequest, dependencies, managedDependencies, + filter); if (singleResult.iterator().hasNext()) { final ArtifactResult first = singleResult.iterator().next(); result = first.getArtifact(); } else { throw new DependencyNotFoundException(String.format("Failed to resolve dependency %s with " - + "dependencyResolver", coordinate)); + + "dependencyResolver", coordinate)); } } } catch (DependencyNotFoundException | DependencyResolverException ex) { @@ -1363,7 +1370,6 @@ private ExceptionCollection collectMavenDependencies(Engine engine, MavenProject // successfully resolved as a reactor dependency - swallow the exception addException = false; } - //CSON: //CSOFF: EmptyBlock if (addException) { if (exCol == null) { exCol = new ExceptionCollection(); @@ -1452,14 +1458,18 @@ && addSnapshotReactorDependency(engine, dependencyNode.getArtifact())) { } return exCol; } + //CSON: OperatorWrap /** * Utility method for a work-around to MSHARED-998 - * @param allDeps The Iterable of the resolved artifacts for all dependencies - * @param theCoord The ArtifactCoordinate of the artifact-with-classifier we intended to resolve + * + * @param allDeps The Iterable of the resolved artifacts for all + * dependencies + * @param theCoord The ArtifactCoordinate of the artifact-with-classifier we + * intended to resolve * @return the resolved artifact matching with {@code theCoord} - * @throws DependencyNotFoundException Not expected to be thrown, but will be thrown if {@code theCoord} could not be - * found within {@code allDeps} + * @throws DependencyNotFoundException Not expected to be thrown, but will + * be thrown if {@code theCoord} could not be found within {@code allDeps} */ private Artifact findClassifierArtifactInAllDeps(final Iterable allDeps, final ArtifactCoordinate theCoord) throws DependencyNotFoundException { @@ -1472,16 +1482,18 @@ private Artifact findClassifierArtifactInAllDeps(final Iterable } if (result == null) { throw new DependencyNotFoundException(String.format("Expected dependency not found in resolved artifacts for " - + "dependency %s", theCoord)); + + "dependency %s", theCoord)); } return result; } /** * Utility method for a work-around to MSHARED-998 + * * @param res A single ArtifactResult obtained from the DependencyResolver * @param theCoord The coordinates of the Artifact that we try to find - * @return {@code true} when theCoord is non-null and matches with the artifact of res + * @return {@code true} when theCoord is non-null and matches with the + * artifact of res */ private boolean sameArtifact(final ArtifactResult res, final ArtifactCoordinate theCoord) { if (res == null || res.getArtifact() == null || theCoord == null) { @@ -2106,13 +2118,18 @@ protected void populateSettings() { settings.setStringIfNotEmpty(Settings.KEYS.DATA_DIRECTORY, dataDirectory); settings.setStringIfNotEmpty(Settings.KEYS.DB_FILE_NAME, dbFilename); - String cveModifiedJson = Optional.ofNullable(cveUrlModified) - .filter(arg -> !arg.isEmpty()) - .orElseGet(this::getDefaultCveUrlModified); + final String cveModifiedJson = Optional.ofNullable(cveUrlModified) + .filter(arg -> !arg.isEmpty()) + .orElseGet(this::getDefaultCveUrlModified); settings.setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON, cveModifiedJson); settings.setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON, cveUrlBase); settings.setStringIfNotEmpty(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, cveWaitTime); settings.setIntIfNotNull(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, cveValidForHours); + if (cveStartYear != null && cveStartYear < 2002) { + getLog().warn("Invalid configuration: cveStartYear must be 2002 or greater"); + cveStartYear = 2002; + } + settings.setIntIfNotNull(Settings.KEYS.CVE_START_YEAR, cveStartYear); settings.setBooleanIfNotNull(Settings.KEYS.PRETTY_PRINT, prettyPrint); artifactScopeExcluded = new ArtifactScopeExcluded(skipTestScope, skipProvidedScope, skipSystemScope, skipRuntimeScope); artifactTypeExcluded = new ArtifactTypeExcluded(skipArtifactType); @@ -2436,8 +2453,8 @@ protected void showSummary(MavenProject mp, Dependency[] dependencies) { } private String getDefaultCveUrlModified() { - return CveUrlParser.newInstance(getSettings()) - .getDefaultCveUrlModified(cveUrlBase); + return CveUrlParser.newInstance(getSettings()) + .getDefaultCveUrlModified(cveUrlBase); } // diff --git a/maven/src/site/markdown/configuration.md b/maven/src/site/markdown/configuration.md index c042d6ae9ea..3e09d89e8dc 100644 --- a/maven/src/site/markdown/configuration.md +++ b/maven/src/site/markdown/configuration.md @@ -138,6 +138,7 @@ cveServerId | The id of a server defined in the settings.xml that c cveUser | The username used when connecting to the cveUrl. Must be empty if cveServerId is specified and should be used. |   | cvePassword | The password used when connecting to the cveUrl. Must be empty if cveServerId is specified and should be used. |   | cveWaitTime | The time in milliseconds to wait between downloads from the NVD. | 4000 | +cveStartYear | The first year of NVD CVE data to download from the NVD. | 2002 suppressionFileServerId | The id of a server defined in the settings.xml that configures the credentials (username and password) for accessing the suppressionFiles. |   | suppressionFileUser | The username used when connecting to the suppressionFiles. Must be empty if suppressionFileServerId is specified and should be used. |   | suppressionFilePassword | The password used when connecting to the suppressionFiles. Must be empty if suppressionFileServerId is specified and should be used. |   | diff --git a/src/main/config/spotbugs_excludes.xml b/src/main/config/spotbugs_excludes.xml index 08653e512e1..5864b605496 100644 --- a/src/main/config/spotbugs_excludes.xml +++ b/src/main/config/spotbugs_excludes.xml @@ -3,10 +3,10 @@ xmlns="https://github.com/spotbugs/filter/3.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://github.com/spotbugs/filter/3.0.0 https://raw.githubusercontent.com/spotbugs/spotbugs/3.1.0/spotbugs/etc/findbugsfilter.xsd"> - + - + @@ -18,4 +18,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/site/markdown/dependency-check-gradle/configuration-aggregate.md b/src/site/markdown/dependency-check-gradle/configuration-aggregate.md index 12ae83ced66..3d1c02f0ad0 100644 --- a/src/site/markdown/dependency-check-gradle/configuration-aggregate.md +++ b/src/site/markdown/dependency-check-gradle/configuration-aggregate.md @@ -87,6 +87,7 @@ Config Group | Property | Description cve | urlModified | URL for the modified CVE JSON data feed. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz | cve | urlBase | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz | cve | waitTime | The time in milliseconds to wait between downloads from the NVD. | 4000 | +cve | startYear | The first year of NVD CVE data to download from the NVD. | 2002 | data | directory | Sets the data directory to hold SQL CVEs contents. This should generally not be changed. |   | data | driver | The name of the database driver. Example: org.h2.Driver. |   | data | driverPath | The path to the database driver JAR file; only used if the driver is not in the class path. |   | diff --git a/src/site/markdown/dependency-check-gradle/configuration-update.md b/src/site/markdown/dependency-check-gradle/configuration-update.md index 3e5ffd1f97a..da9ca8de3ac 100644 --- a/src/site/markdown/dependency-check-gradle/configuration-update.md +++ b/src/site/markdown/dependency-check-gradle/configuration-update.md @@ -68,6 +68,7 @@ Config Group | Property | Description cve | urlModified | URL for the modified CVE JSON data feed. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz | cve | urlBase | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz | cve | waitTime | The time in milliseconds to wait between downloads from the NVD. | 4000 | +cve | startYear | The first year of NVD CVE data to download from the NVD. | 2002 | data | directory | Sets the data directory to hold SQL CVEs contents. This should generally not be changed. |   | data | driver | The name of the database driver. Example: org.h2.Driver. |   | data | driverPath | The path to the database driver JAR file; only used if the driver is not in the class path. |   | diff --git a/src/site/markdown/dependency-check-gradle/configuration.md b/src/site/markdown/dependency-check-gradle/configuration.md index 38503983daa..413b8368419 100644 --- a/src/site/markdown/dependency-check-gradle/configuration.md +++ b/src/site/markdown/dependency-check-gradle/configuration.md @@ -71,6 +71,7 @@ Config Group | Property | Description cve | urlModified | URL for the modified CVE JSON data feed. When mirroring the NVD you must mirror the *.json.gz and the *.meta files. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz | cve | urlBase | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz | cve | waitTime | The time in milliseconds to wait between downloads from the NVD. | 4000 | +cve | startYear | The first year of NVD CVE data to download from the NVD. | 2002 | data | directory | Sets the data directory to hold SQL CVEs contents. This should generally not be changed. |   | data | driver | The name of the database driver. Example: org.h2.Driver. |   | data | driverPath | The path to the database driver JAR file; only used if the driver is not in the class path. |   | diff --git a/test-docker.sh b/test-docker.sh index 46247afc4eb..57d6fb3e500 100755 --- a/test-docker.sh +++ b/test-docker.sh @@ -65,7 +65,9 @@ docker run --rm \ --format "JSON" \ --project "test scan" \ --out /report \ - --log /report/odc.log + --log /report/odc.log \ + --cveDownloadWait 20000 \ + --cveStartYear 2020 # return to original working directory cd - diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/CveUrlParser.java b/utils/src/main/java/org/owasp/dependencycheck/utils/CveUrlParser.java index e9bbb72fd29..53f5c61ea0e 100644 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/CveUrlParser.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/CveUrlParser.java @@ -1,3 +1,20 @@ +/* + * This file is part of dependency-check-ant. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2021 The OWASP Foundation. All Rights Reserved. + */ package org.owasp.dependencycheck.utils; /** @@ -10,10 +27,22 @@ */ public interface CveUrlParser { - static CveUrlParser newInstance(Settings settings) { - return new DefaultCveUrlModifiedParser(settings); - } + /** + * Create a new instance of the CveUrlParser. + * + * @param settings the configured settings + * @return the new instance + */ + static CveUrlParser newInstance(Settings settings) { + return new DefaultCveUrlModifiedParser(settings); + } - String getDefaultCveUrlModified(String baseUrl); + /** + * Gets the default CVE Modified URL. + * + * @param baseUrl the base CVE URL + * @return the default CVE Modified URL + */ + String getDefaultCveUrlModified(String baseUrl); } diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/DefaultCveUrlModifiedParser.java b/utils/src/main/java/org/owasp/dependencycheck/utils/DefaultCveUrlModifiedParser.java index 5e52e97b668..f6b827aaa0c 100644 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/DefaultCveUrlModifiedParser.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/DefaultCveUrlModifiedParser.java @@ -1,3 +1,20 @@ +/* + * This file is part of dependency-check-ant. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2021 The OWASP Foundation. All Rights Reserved. + */ package org.owasp.dependencycheck.utils; import java.util.Objects; @@ -10,21 +27,31 @@ */ public final class DefaultCveUrlModifiedParser implements CveUrlParser { - private static final String URL_SEPARATOR = "/"; - - private final Settings settings; + /** + * The URL separator character. + */ + private static final String URL_SEPARATOR = "/"; + /** + * The configured ODC settings. + */ + private final Settings settings; - DefaultCveUrlModifiedParser(Settings settings) { - this.settings = settings; - } + /** + * Constructs the default CVE Modified URL parser. + * + * @param settings the configured settings + */ + DefaultCveUrlModifiedParser(Settings settings) { + this.settings = settings; + } - @Override - public String getDefaultCveUrlModified(String baseUrl) { - String defaultBaseUrlEnd = URL_SEPARATOR + settings.getString(Settings.KEYS.CVE_BASE_DEFAULT_FILENAME); - if (Objects.nonNull(baseUrl) && baseUrl.endsWith(defaultBaseUrlEnd)) { - String defaultModifiedUrlEnd = URL_SEPARATOR + settings.getString(Settings.KEYS.CVE_MODIFIED_DEFAULT_FILENAME); - return baseUrl.substring(0, baseUrl.length() - defaultBaseUrlEnd.length()) + defaultModifiedUrlEnd; + @Override + public String getDefaultCveUrlModified(String baseUrl) { + final String defaultBaseUrlEnd = URL_SEPARATOR + settings.getString(Settings.KEYS.CVE_BASE_DEFAULT_FILENAME); + if (Objects.nonNull(baseUrl) && baseUrl.endsWith(defaultBaseUrlEnd)) { + final String defaultModifiedUrlEnd = URL_SEPARATOR + settings.getString(Settings.KEYS.CVE_MODIFIED_DEFAULT_FILENAME); + return baseUrl.substring(0, baseUrl.length() - defaultBaseUrlEnd.length()) + defaultModifiedUrlEnd; + } + return null; } - return null; - } }