Create SUSE OVAL importer

setup.cfg

@@ -100,7 +100,7 @@ install_requires =
 dev =
     # Validation
     pycodestyle>=2.8.0
-    black>=22.3.0
+    black==22.3.0
     isort>=5.10.1
     doc8>=0.11.1
     # Documentation

vulnerabilities/importer.py

@@ -429,47 +429,54 @@ def get_data_from_xml_doc(
         for definition_data in raw_data:
             # These fields are definition level, i.e common for all elements
             # connected/linked to an OvalDefinition
-            vuln_id = definition_data["vuln_id"]
-            description = definition_data["description"]
-            severities = []
-            severity = definition_data.get("severity")
-            if severity:
-                severities.append(
-                    VulnerabilitySeverity(system=severity_systems.GENERIC, value=severity)
-                )
-            references = [
-                Reference(url=url, severities=severities)
-                for url in definition_data["reference_urls"]
-            ]
-            affected_packages = []
-            for test_data in definition_data["test_data"]:
-                for package_name in test_data["package_list"]:
-                    affected_version_range = test_data["version_ranges"]
-                    vrc = RANGE_CLASS_BY_SCHEMES[pkg_metadata["type"]]
-                    if affected_version_range:
-                        try:
-                            affected_version_range = vrc.from_native(affected_version_range)
-                        except Exception as e:
-                            logger.error(
-                                f"Failed to parse version range {affected_version_range!r} "
-                                f"for package {package_name!r}:\n{e}\n"
-                                f"{definition_data!r}"
-                            )
-                            continue
-                    if package_name:
-                        affected_packages.append(
-                            AffectedPackage(
-                                package=self.create_purl(package_name, pkg_metadata),
-                                affected_version_range=affected_version_range,
+
+            # NOTE: This is where we loop through the list of CVEs/aliases.
+            vuln_id_list = definition_data["vuln_id"]
+
+            for vuln_id_item in vuln_id_list:
+                vuln_id = vuln_id_item
+                description = definition_data["description"]
+
+                severities = []
+                severity = definition_data.get("severity")
+                if severity:
+                    severities.append(
+                        VulnerabilitySeverity(system=severity_systems.GENERIC, value=severity)
+                    )
+                references = [
+                    Reference(url=url, severities=severities)
+                    for url in definition_data["reference_urls"]
+                ]
+                affected_packages = []
+
+                for test_data in definition_data["test_data"]:
+                    for package_name in test_data["package_list"]:
+                        affected_version_range = test_data["version_ranges"]
+                        vrc = RANGE_CLASS_BY_SCHEMES[pkg_metadata["type"]]
+                        if affected_version_range:
+                            try:
+                                affected_version_range = vrc.from_native(affected_version_range)
+                            except Exception as e:
+                                logger.error(
+                                    f"Failed to parse version range {affected_version_range!r} "
+                                    f"for package {package_name!r}:\n{e}"
+                                )
+                                continue
+                        if package_name:
+                            affected_packages.append(
+                                AffectedPackage(
+                                    package=self.create_purl(package_name, pkg_metadata),
+                                    affected_version_range=affected_version_range,
+                                )
                             )
-                        )
-            date_published = dateparser.parse(timestamp)
-            if not date_published.tzinfo:
-                date_published = date_published.replace(tzinfo=pytz.UTC)
-            yield AdvisoryData(
-                aliases=[vuln_id],
-                summary=description,
-                affected_packages=affected_packages,
-                references=sorted(references),
-                date_published=date_published,
-            )
+
+                date_published = dateparser.parse(timestamp)
+                if not date_published.tzinfo:
+                    date_published = date_published.replace(tzinfo=pytz.UTC)
+                yield AdvisoryData(
+                    aliases=[vuln_id],
+                    summary=description,
+                    affected_packages=sorted(affected_packages),
+                    references=sorted(references),
+                    date_published=date_published,
+                )

vulnerabilities/importers/suse_oval.py

@@ -0,0 +1,69 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+
+import gzip
+import xml.etree.ElementTree as ET
+
+import requests
+from bs4 import BeautifulSoup
+
+from vulnerabilities.importer import OvalImporter
+
+
+class SuseOvalImporter(OvalImporter):
+    spdx_license_expression = "CC-BY-4.0"
+    license_url = "https://ftp.suse.com/pub/projects/security/oval/LICENSE"
+    base_url = "https://ftp.suse.com/pub/projects/security/oval/"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.translations = {"less than": "<", "equals": "=", "greater than or equal": ">="}
+
+    def _fetch(self):
+        page = requests.get(self.base_url).text
+        soup = BeautifulSoup(page, "lxml")
+
+        suse_oval_files = [
+            self.base_url + node.get("href")
+            for node in soup.find_all("a")
+            if node.get("href").endswith(".gz")
+        ]
+
+        for suse_file in filter(suse_oval_files):
+            response = requests.get(suse_file)
+
+            extracted = gzip.decompress(response.content)
+            yield (
+                {"type": "rpm", "namespace": "opensuse"},
+                ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))),
+            )
+
+
+def filter(suse_oval_files):
+    """
+    Filter to exclude "name.xml" when we also have "name-affected.xml", e.g.,
+    "opensuse.leap.15.3.xml.gz" vs. "opensuse.leap.15.3-affected.xml.gz".  See
+    https://ftp.suse.com/pub/projects/security/oval/README: "name-affected.xml" includes
+    "fixed security issues and the analyzed issues both affecting and NOT affecting SUSE" and
+    "name.xml" includes "fixed security issues and the analyzed issues NOT affecting SUSE."
+    """
+    affected_files = [
+        affected_file for affected_file in suse_oval_files if "-affected" in affected_file
+    ]
+
+    trimmed_affected_files = [
+        affected_file.replace("-affected", "") for affected_file in affected_files
+    ]
+
+    filtered_suse_oval_files = [
+        gz_file for gz_file in suse_oval_files if gz_file not in trimmed_affected_files
+    ]
+
+    return filtered_suse_oval_files

vulnerabilities/oval_parser.py

@@ -25,7 +25,6 @@
 
 class OvalParser:
     def __init__(self, translations: Dict, oval_document: ET.ElementTree):
-
         self.translations = translations
         self.oval_document = OvalDocument(oval_document)
         self.all_definitions = self.oval_document.getDefinitions()
@@ -37,17 +36,14 @@ def get_data(self) -> List[Dict]:
         """
         oval_data = []
         for definition in self.all_definitions:
-
             matching_tests = self.get_tests_of_definition(definition)
             if not matching_tests:
                 continue
             definition_data = {"test_data": []}
             # TODO:this could use some data cleaning
             definition_data["description"] = definition.getMetadata().getDescription() or ""
-
             definition_data["vuln_id"] = self.get_vuln_id_from_definition(definition)
             definition_data["reference_urls"] = self.get_urls_from_definition(definition)
-
             definition_data["severity"] = self.get_severity_from_definition(definition)
 
             for test in matching_tests:
@@ -72,24 +68,30 @@ def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]:
         criteria_refs = []
 
         for child in definition.element.iter():
-
             if "test_ref" in child.attrib:
                 criteria_refs.append(child.get("test_ref"))
 
         matching_tests = []
         for ref in criteria_refs:
             oval_test = self.oval_document.getElementByID(ref)
+            # All matches will be `rpminfo_test` elements inside the `tests` element.
+            # Test for len == 2 because this IDs a pair of nested `object` and `state` elements.
             if len(oval_test.element) == 2:
                 _, state = self.get_object_state_of_test(oval_test)
                 valid_test = True
                 for child in state.element:
                     if child.get("operation") not in self.translations:
                         valid_test = False
-                        break
-                if valid_test:
-                    matching_tests.append(self.oval_document.getElementByID(ref))
+                        continue
+                    elif (
+                        child.get("operation") in self.translations
+                        # "debian_evr_string" is used in both Debian and Ubuntu test XML files; SUSE OVAL uses "evr_string".
+                        # See also https://github.com/OVALProject/Language/blob/master/docs/oval-common-schema.md
+                        and child.get("datatype") in ["evr_string", "debian_evr_string"]
+                    ):
+                        matching_tests.append(self.oval_document.getElementByID(ref))
 
-        return matching_tests
+        return list(set(matching_tests))
 
     def get_object_state_of_test(self, test: OvalTest) -> Tuple[OvalObject, OvalState]:
         """
@@ -109,6 +111,7 @@ def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]:
         pkg_list = []
 
         for var in obj.element:
+            # It appears that `var_ref` is used in Ubuntu OVAL but not Debian or SUSE.
             if var.get("var_ref"):
                 var_elem = self.oval_document.getElementByID(var.get("var_ref"))
                 comment = var_elem.element.get("comment")
@@ -178,9 +181,18 @@ def get_severity_from_definition(definition: OvalDefinition) -> Set[str]:
 
     @staticmethod
     def get_vuln_id_from_definition(definition):
-        # SUSE and Ubuntu OVAL files will get cves via this loop
+        # SUSE and Ubuntu OVAL files will get CVEs via this loop.
+        cve_list = []
         for child in definition.element.iter():
-            if child.get("ref_id"):
-                return child.get("ref_id")
-        # Debian OVAL files will get cves via this
-        return definition.getMetadata().getTitle()
+            if child.get("ref_id") and child.get("source"):
+                if child.get("source") == "CVE":
+                    if not child.get("ref_id").startswith("CVE"):
+                        unwanted_prefix = child.get("ref_id").split("CVE")[0]
+                        cve_list.append(child.get("ref_id").replace(unwanted_prefix, ""))
+                    else:
+                        cve_list.append(child.get("ref_id"))
+        # Debian OVAL files (no "ref_id") will get CVEs via this.
+        if len(cve_list) == 0:
+            cve_list.append(definition.getMetadata().getTitle())
+
+        return cve_list

vulnerabilities/tests/test_data/suse_oval/mock-definitions-only.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<oval_definitions xsi:schemaLocation="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux linux-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#unix unix-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-common-5 oval-common-schema.xsd"
+    xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xmlns:oval="http://oval.mitre.org/XMLSchema/oval-common-5"
+    xmlns:oval-def="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+    <definitions>
+        <definition id="oval:org.opensuse.security:def:2009030400" version="1" class="patch">
+            <metadata>
+                <title>CVE-2008-5679</title>
+                <affected family="unix">
+                </affected>
+                <reference ref_id="CVE-2008-5679" ref_url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2008-5679" source="CVE"/>
+                <description>
+	The HTML parsing engine in Opera before 9.63 allows remote attackers to execute arbitrary code via crafted web pages that trigger an invalid pointer calculation and heap corruption.
+                </description>
+            </metadata>
+        </definition>
+        <definition id="oval:org.opensuse.security:def:2009030400" version="1" class="patch">
+            <metadata>
+                <title>foobar-CVE-1234-5678</title>
+                <affected family="unix">
+                </affected>
+                <reference ref_id="foobar-CVE-1234-5678" ref_url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=foobar-CVE-1234-5678" source="CVE"/>
+                <description>
+            Blah blah blah.
+                </description>
+            </metadata>
+        </definition>
+        <definition id="oval:org.opensuse.security:def:2009030400" version="1" class="patch">
+            <metadata>
+                <title>nonesuchCVE-1111-2222</title>
+                <affected family="unix">
+                </affected>
+                <reference ref_id="nonesuchCVE-1111-2222" ref_url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=nonesuchCVE-1111-2222" source="CVE"/>
+                <description>
+                    Blah blah blah.
+                </description>
+            </metadata>
+        </definition>
+    </definitions>
+</oval_definitions>

vulnerabilities/tests/test_data/suse_oval/org.opensuse.CVE-2008-5679.xml

@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<oval_definitions
+	xsi:schemaLocation="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux linux-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#unix unix-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-common-5 oval-common-schema.xsd"
+	xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xmlns:oval="http://oval.mitre.org/XMLSchema/oval-common-5"
+	xmlns:oval-def="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+  <generator>
+      <oval:product_name>Marcus OVAL Generator</oval:product_name>
+      <oval:schema_version>5.5</oval:schema_version>
+      <oval:timestamp>2009-01-14T09:08:29.480-05:00</oval:timestamp>
+  </generator>
+  <definitions>
+
+<definition id="oval:org.opensuse.security:def:2009030400" version="1" class="patch">
+ <metadata>
+ <title>CVE-2008-5679</title>
+	<affected family="unix">
+	</affected>
+	<reference ref_id="CVE-2008-5679" ref_url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2008-5679" source="CVE"/>
+	<description>
+	The HTML parsing engine in Opera before 9.63 allows remote attackers to execute arbitrary code via crafted web pages that trigger an invalid pointer calculation and heap corruption.
+	</description>
+ </metadata>
+	<criteria operator="OR">
+		<criteria operator="AND">
+			<criteria operator="OR">
+			<criterion test_ref="oval:org.opensuse.security:tst:2009030401" comment="suse103 is installed"/>
+			<criterion test_ref="oval:org.opensuse.security:tst:2009030402" comment="suse110 is installed"/>
+			<criterion test_ref="oval:org.opensuse.security:tst:2009030403" comment="suse111 is installed"/>
+			</criteria>
+			<criterion test_ref="oval:org.opensuse.security:tst:2009030400" comment="oval:org.opensuse.security:tst:2009030400 is installed"/>
+		</criteria>
+		<criteria operator="AND">
+			<criterion test_ref="oval:org.opensuse.security:tst:2009030403" comment="suse111-debug is installed"/>
+			<criterion test_ref="oval:org.opensuse.security:tst:2009030400" comment="oval:org.opensuse.security:tst:2009030400 is installed"/>
+		</criteria>
+	</criteria>
+</definition>
+ </definitions>
+ <tests>
+	<rpminfo_test id="oval:org.opensuse.security:tst:2009030400" version="1" comment="oval:org.opensuse.security:obj:2009030400 is version oval:org.opensuse.security:ste:2009030400" check="at least one" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+		<object object_ref="oval:org.opensuse.security:obj:2009030400"/>
+		<state state_ref="oval:org.opensuse.security:ste:2009030400"/>
+	</rpminfo_test>
+	<rpminfo_test id="oval:org.opensuse.security:tst:2009030401" version="1" comment="oval:org.opensuse.security:obj:2009030401 is version oval:org.opensuse.security:ste:2009030401" check="at least one" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+		<object object_ref="oval:org.opensuse.security:obj:2009030401"/>
+		<state state_ref="oval:org.opensuse.security:ste:2009030401"/>
+	</rpminfo_test>
+	<rpminfo_test id="oval:org.opensuse.security:tst:2009030402" version="1" comment="oval:org.opensuse.security:obj:2009030401 is version oval:org.opensuse.security:ste:2009030402" check="at least one" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+		<object object_ref="oval:org.opensuse.security:obj:2009030401"/>
+		<state state_ref="oval:org.opensuse.security:ste:2009030402"/>
+	</rpminfo_test>
+	<rpminfo_test id="oval:org.opensuse.security:tst:2009030403" version="1" comment="oval:org.opensuse.security:obj:2009030401 is version oval:org.opensuse.security:ste:2009030403" check="at least one" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+		<object object_ref="oval:org.opensuse.security:obj:2009030401"/>
+		<state state_ref="oval:org.opensuse.security:ste:2009030403"/>
+	</rpminfo_test>
+ </tests>
+ <objects>
+	<rpminfo_object id="oval:org.opensuse.security:obj:2009030400" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+		<name>opera</name>
+	</rpminfo_object>
+	<rpminfo_object id="oval:org.opensuse.security:obj:2009030401" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+		<name>openSUSE-release</name>
+	</rpminfo_object>
+ </objects>
+ <states>
+  <rpminfo_state id="oval:org.opensuse.security:ste:2009030400" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+   <evr datatype="evr_string" operation="less than">0:9.63-1.1</evr>
+  </rpminfo_state>
+  <rpminfo_state id="oval:org.opensuse.security:ste:2009030401" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+   <version operation="pattern match">^10.3$</version>
+  </rpminfo_state>
+  <rpminfo_state id="oval:org.opensuse.security:ste:2009030402" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+   <version operation="pattern match">^11.0$</version>
+  </rpminfo_state>
+  <rpminfo_state id="oval:org.opensuse.security:ste:2009030403" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+   <version operation="pattern match">^11.1$</version>
+  </rpminfo_state>
+ </states>
+</oval_definitions>