From 3319c3178b2c9c283c42d5577a4bf9bff5503143 Mon Sep 17 00:00:00 2001 From: Andres Tito Date: Wed, 11 Dec 2024 13:35:09 +0000 Subject: [PATCH 1/3] feat: Add option to download vulnerabilities from VulnDB without NVD matches To prevent duplicates in Dependency-Track, this update introduces the ability to download vulnerabilities from VulnDB that lack NVD matches. This helps fill the gap in NVD reports by ensuring only unique vulnerabilities are processed. New Option: --mirror-vulnerabilities --no-nvd-additional Details: - Utilizes the VulnDB property `nvd_additional_information`. - Checks if the `nvd_additional_information` array size is 0 to filter vulnerabilities without NVD matches. Signed-off-by: Andres Tito --- pom.xml | 6 ++ .../vulndbdatamirror/VulnDbDataMirror.java | 88 ++++++++++++++++--- .../vulndbdatamirror/client/VulnDbApi.java | 3 +- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 611b5a1..e4982ca 100644 --- a/pom.xml +++ b/pom.xml @@ -123,6 +123,12 @@ ${lib.unirest.version} compile + + com.fasterxml.jackson.core + jackson-databind + 2.15.2 + + diff --git a/src/main/java/us/springett/vulndbdatamirror/VulnDbDataMirror.java b/src/main/java/us/springett/vulndbdatamirror/VulnDbDataMirror.java index f9ced09..cfd4442 100644 --- a/src/main/java/us/springett/vulndbdatamirror/VulnDbDataMirror.java +++ b/src/main/java/us/springett/vulndbdatamirror/VulnDbDataMirror.java @@ -15,6 +15,15 @@ */ package us.springett.vulndbdatamirror; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Date; +import java.util.Properties; + import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; @@ -24,17 +33,15 @@ import org.apache.commons.cli.ParseException; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + import us.springett.vulndbdatamirror.client.VulnDbApi; import us.springett.vulndbdatamirror.parser.model.Results; import us.springett.vulndbdatamirror.parser.model.Status; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Date; -import java.util.Properties; /** * This self-contained class can be called from the command-line. It downloads the @@ -62,6 +69,7 @@ public static void main (String[] args) throws Exception { options.addOption( "vend", "mirror-vendors", false, "Mirror the vendors data feed" ); options.addOption( "prod", "mirror-products", false, "Mirror the products data feed" ); options.addOption( "vuln", "mirror-vulnerabilities", false, "Mirror the vulnerabilities data feed" ); + options.addOption("n","no-nvd-additional",false,"Download vulnerabilities without NVD additional information only"); options.addOption(Option.builder().longOpt("consumer-key").desc("The Consumer Key provided by VulnDB") .hasArg().required().argName("key").build() @@ -95,7 +103,11 @@ public static void main (String[] args) throws Exception { mirror.mirrorProducts(); } if (line.hasOption("mirror-vulnerabilities")) { - mirror.mirrorVulnerabilities(); + if (line.hasOption("no-nvd-additional")) { + mirror.mirrorVulnerabilitiesWithFilter(); + } else { + mirror.mirrorVulnerabilities(); + } } if (line.hasOption("mirror-update")) { System.out.println("Fetching last updating vulnerabilities"); @@ -208,6 +220,56 @@ private void mirrorVulnerabilities() throws Exception { System.exit(1); } } + private void mirrorVulnerabilitiesWithFilter() throws Exception { + final VulnDbApi api = new VulnDbApi(this.consumerKey, this.consumerSecret); + System.out.print("\nMirroring Vulnerabilities without NVD Additional Information feed...\n"); + int page = lastSuccessfulPage("vulnerabilities_filtered"); + boolean more = true; + while (more) { + final Results results = api.getVulnerabilities(100, page); + if (results.isSuccessful()) { + final String filteredResults = filterNVDAdditionalInfo(results.getRawResults()); + if (!filteredResults.isEmpty()) { + FileUtils.writeStringToFile(new File(this.outputDir, "vulnerabilities_filtered_" + results.getPage() + ".json"), filteredResults, "UTF-8"); + storeSuccessfulPage(VulnDbApi.Type.VULNERABILITIES_FILTERED, results.getPage()); + } else { + System.out.println("No vulnerabilities without NVD additional information found on page " + page); + } + more = results.getPage() * 100 < results.getTotal(); + page++; + } else { + System.exit(1); + } + } + if (downloadFailed) { + System.exit(1); + } + } + + private String filterNVDAdditionalInfo(String jsonResults) throws Exception { + + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = mapper.readTree(jsonResults); + ArrayNode resultsNode = (ArrayNode) rootNode.get("results"); + ArrayNode filteredNode = mapper.createArrayNode(); + + System.out.print("\nFILTERING...\n"); + for (JsonNode result : resultsNode) { + JsonNode nvdAdditionalInfo = result.get("nvd_additional_information"); + System.out.print(nvdAdditionalInfo); + System.out.print("\n\n"); + if (nvdAdditionalInfo != null && nvdAdditionalInfo.isArray() && nvdAdditionalInfo.size() == 0) { + filteredNode.add(result); + } + } + + ObjectNode filteredResultsNode = mapper.createObjectNode(); + filteredResultsNode.set("results", filteredNode); + filteredResultsNode.put("total_entries", rootNode.get("total_entries").asInt()); + filteredResultsNode.put("current_page", rootNode.get("current_page").asInt()); + + return mapper.writeValueAsString(filteredResultsNode); + } private void mirrorUpdatedVulnerabilities(final int hours_ago) throws Exception { final VulnDbApi api = new VulnDbApi(this.consumerKey, this.consumerSecret); @@ -229,11 +291,17 @@ private void mirrorUpdatedVulnerabilities(final int hours_ago) throws Exception } private int lastSuccessfulPage(String prefix) { + if (prefix.equals("vulnerabilities_filtered")) { + prefix = "vulnerabilities_filtered"; // Adjust property key as needed + } return Integer.parseInt(properties.getProperty(prefix + ".last_success_page", "1")); } private void storeSuccessfulPage(VulnDbApi.Type type, int page) { - final String prefix = type.name().toLowerCase(); + String prefix = type.name().toLowerCase(); + if (type == VulnDbApi.Type.VULNERABILITIES_FILTERED) { // Assuming you'll add this to your VulnDbApi.Type enum + prefix = "vulnerabilities_filtered"; + } properties.setProperty(prefix + ".last_success_page", String.valueOf(page)); properties.setProperty(prefix + ".last_success_timestamp", String.valueOf(new Date().getTime())); writeProperties(); diff --git a/src/main/java/us/springett/vulndbdatamirror/client/VulnDbApi.java b/src/main/java/us/springett/vulndbdatamirror/client/VulnDbApi.java index 99ef753..33a6fcd 100644 --- a/src/main/java/us/springett/vulndbdatamirror/client/VulnDbApi.java +++ b/src/main/java/us/springett/vulndbdatamirror/client/VulnDbApi.java @@ -62,7 +62,8 @@ public class VulnDbApi { public enum Type { VENDORS, PRODUCTS, - VULNERABILITIES + VULNERABILITIES, + VULNERABILITIES_FILTERED } public VulnDbApi(final String consumerKey, final String consumerSecret) { From 22ba0d721858a250207702c8d8c02acb6c936c3f Mon Sep 17 00:00:00 2001 From: Andres Tito Date: Wed, 11 Dec 2024 13:39:28 +0000 Subject: [PATCH 2/3] fix: Correct logical condition for feed mirroring default behavior The previous condition used `&&` (AND), causing the default mirroring behavior to trigger even when only one feed option was specified. This led to unexpected behavior where all feeds were mirrored despite a valid single feed selection. Updated the condition to use `||` (OR), ensuring the default behavior only occurs when *none* of the feed options (`--mirror-vendors`, `--mirror-products`, `--mirror-vulnerabilities`) are specified. This change prevents unnecessary mirroring and ensures the specified feed options are respected. Signed-off-by: Andres Tito --- .../java/us/springett/vulndbdatamirror/VulnDbDataMirror.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/us/springett/vulndbdatamirror/VulnDbDataMirror.java b/src/main/java/us/springett/vulndbdatamirror/VulnDbDataMirror.java index cfd4442..a002481 100644 --- a/src/main/java/us/springett/vulndbdatamirror/VulnDbDataMirror.java +++ b/src/main/java/us/springett/vulndbdatamirror/VulnDbDataMirror.java @@ -114,7 +114,7 @@ public static void main (String[] args) throws Exception { final int hours = Integer.parseInt(line.getOptionValue("mirror-update")); mirror.mirrorUpdatedVulnerabilities(hours); } - if (!(line.hasOption("mirror-vendors") && line.hasOption("mirror-products") && line.hasOption("mirror-vulnerabilities"))) { + if (!(line.hasOption("mirror-vendors") || line.hasOption("mirror-products") || line.hasOption("mirror-vulnerabilities"))) { System.out.println("A feed to mirror was not specified. Defaulting to mirror all feeds."); mirror.mirrorVendors(); mirror.mirrorProducts(); From e610cb41ad80acbbf3078b81002c3b7197ca82af Mon Sep 17 00:00:00 2001 From: Andres Tito Date: Wed, 11 Dec 2024 14:05:08 +0000 Subject: [PATCH 3/3] Update usage section from README Signed-off-by: Andres Tito --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 1065af8..e5af0c6 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,9 @@ usage: vulndb-data-mirror -vend,--mirror-vendors Mirror the vendors data feed -vuln,--mirror-vulnerabilities Mirror the vulnerabilities data feed -stat,--status-only Displays VulnDB API status only + -u,--mirror-update Mirror just the updated vulnerabilities data feed from the past + -n, --no-nvd-additional Filters vulnerabilities to download only those without additional information from the NVD. + **Note:** This option only works when used with `--mirror-vulnerabilities`. ``` ### Mirror Recovery