Create SUSE OVAL importer (#1085)

* Create suse_oval.py and related test files #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Explore OvalParser() parsing process #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Commit latest parsing changes #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Update tests #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Add note re loop through list of aliases #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Work on alias/CVE loop #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Add OVAL parsing test #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Refactor OVAL-relared code, fix failing tests #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Delete unneeded large XML files #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Filter for name-affected.xml files, check CVE prefixes #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Refactor OvalParser(), add and update tests #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Refactor tests and test files, freeze Black version #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Create suse_oval.py and related test files #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Explore OvalParser() parsing process #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Commit latest parsing changes #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Update tests #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Add note re loop through list of aliases #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Work on alias/CVE loop #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Add OVAL parsing test #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Refactor OVAL-relared code, fix failing tests #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Delete unneeded large XML files #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Filter for name-affected.xml files, check CVE prefixes #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Refactor OvalParser(), add and update tests #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Refactor tests and test files, freeze Black version #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Update setup.cfg * Replace 'list()' with 'sorted()' #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Modify OvalElement class __lt__ method and create test #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Add 'url' field to expected JSON test output #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> * Update __lt__ method and related test #1079 Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com> --------- Signed-off-by: John M. Horan <johnmhoran@gmail.com> Signed-off-by: John M. Horan johnmhoran@gmail.com Co-authored-by: Tushar Goel <34160672+TG1999@users.noreply.github.com>
aboutcode-org · Dec 29, 2023 · 02e3fae · 02e3fae
1 parent 5932722
commit 02e3fae
Show file tree

Hide file tree

Showing 10 changed files with 587 additions and 68 deletions.
diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py
@@ -75,7 +75,6 @@ def from_dict(cls, severity: dict):
 
 @dataclasses.dataclass(order=True)
 class Reference:
-
     reference_id: str = ""
     url: str = ""
     severities: List[VulnerabilitySeverity] = dataclasses.field(default_factory=list)
@@ -437,48 +436,55 @@ def get_data_from_xml_doc(
         for definition_data in raw_data:
             # These fields are definition level, i.e common for all elements
             # connected/linked to an OvalDefinition
-            vuln_id = definition_data["vuln_id"]
-            description = definition_data["description"]
-            severities = []
-            severity = definition_data.get("severity")
-            if severity:
-                severities.append(
-                    VulnerabilitySeverity(system=severity_systems.GENERIC, value=severity)
-                )
-            references = [
-                Reference(url=url, severities=severities)
-                for url in definition_data["reference_urls"]
-            ]
-            affected_packages = []
-            for test_data in definition_data["test_data"]:
-                for package_name in test_data["package_list"]:
-                    affected_version_range = test_data["version_ranges"]
-                    vrc = RANGE_CLASS_BY_SCHEMES[pkg_metadata["type"]]
-                    if affected_version_range:
-                        try:
-                            affected_version_range = vrc.from_native(affected_version_range)
-                        except Exception as e:
-                            logger.error(
-                                f"Failed to parse version range {affected_version_range!r} "
-                                f"for package {package_name!r}:\n{e}\n"
-                                f"{definition_data!r}"
-                            )
-                            continue
-                    if package_name:
-                        affected_packages.append(
-                            AffectedPackage(
-                                package=self.create_purl(package_name, pkg_metadata),
-                                affected_version_range=affected_version_range,
+
+            # NOTE: This is where we loop through the list of CVEs/aliases.
+            vuln_id_list = definition_data["vuln_id"]
+
+            for vuln_id_item in vuln_id_list:
+                vuln_id = vuln_id_item
+                description = definition_data["description"]
+
+                severities = []
+                severity = definition_data.get("severity")
+                if severity:
+                    severities.append(
+                        VulnerabilitySeverity(system=severity_systems.GENERIC, value=severity)
+                    )
+                references = [
+                    Reference(url=url, severities=severities)
+                    for url in definition_data["reference_urls"]
+                ]
+                affected_packages = []
+
+                for test_data in definition_data["test_data"]:
+                    for package_name in test_data["package_list"]:
+                        affected_version_range = test_data["version_ranges"]
+                        vrc = RANGE_CLASS_BY_SCHEMES[pkg_metadata["type"]]
+                        if affected_version_range:
+                            try:
+                                affected_version_range = vrc.from_native(affected_version_range)
+                            except Exception as e:
+                                logger.error(
+                                    f"Failed to parse version range {affected_version_range!r} "
+                                    f"for package {package_name!r}:\n{e}"
+                                )
+                                continue
+                        if package_name:
+                            affected_packages.append(
+                                AffectedPackage(
+                                    package=self.create_purl(package_name, pkg_metadata),
+                                    affected_version_range=affected_version_range,
+                                )
                             )
-                        )
-            date_published = dateparser.parse(timestamp)
-            if not date_published.tzinfo:
-                date_published = date_published.replace(tzinfo=pytz.UTC)
-            yield AdvisoryData(
-                aliases=[vuln_id],
-                summary=description,
-                affected_packages=affected_packages,
-                references=sorted(references),
-                date_published=date_published,
-                url=self.data_url,
-            )
+
+                date_published = dateparser.parse(timestamp)
+                if not date_published.tzinfo:
+                    date_published = date_published.replace(tzinfo=pytz.UTC)
+                yield AdvisoryData(
+                    aliases=[vuln_id],
+                    summary=description,
+                    affected_packages=sorted(affected_packages),
+                    references=sorted(references),
+                    date_published=date_published,
+                    url=self.data_url,
+                )
diff --git a/vulnerabilities/importers/suse_oval.py b/vulnerabilities/importers/suse_oval.py
@@ -0,0 +1,69 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+
+import gzip
+import xml.etree.ElementTree as ET
+
+import requests
+from bs4 import BeautifulSoup
+
+from vulnerabilities.importer import OvalImporter
+
+
+class SuseOvalImporter(OvalImporter):
+    spdx_license_expression = "CC-BY-4.0"
+    license_url = "https://ftp.suse.com/pub/projects/security/oval/LICENSE"
+    base_url = "https://ftp.suse.com/pub/projects/security/oval/"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.translations = {"less than": "<", "equals": "=", "greater than or equal": ">="}
+
+    def _fetch(self):
+        page = requests.get(self.base_url).text
+        soup = BeautifulSoup(page, "lxml")
+
+        suse_oval_files = [
+            self.base_url + node.get("href")
+            for node in soup.find_all("a")
+            if node.get("href").endswith(".gz")
+        ]
+
+        for suse_file in filter(suse_oval_files):
+            response = requests.get(suse_file)
+
+            extracted = gzip.decompress(response.content)
+            yield (
+                {"type": "rpm", "namespace": "opensuse"},
+                ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))),
+            )
+
+
+def filter(suse_oval_files):
+    """
+    Filter to exclude "name.xml" when we also have "name-affected.xml", e.g.,
+    "opensuse.leap.15.3.xml.gz" vs. "opensuse.leap.15.3-affected.xml.gz".  See
+    https://ftp.suse.com/pub/projects/security/oval/README: "name-affected.xml" includes
+    "fixed security issues and the analyzed issues both affecting and NOT affecting SUSE" and
+    "name.xml" includes "fixed security issues and the analyzed issues NOT affecting SUSE."
+    """
+    affected_files = [
+        affected_file for affected_file in suse_oval_files if "-affected" in affected_file
+    ]
+
+    trimmed_affected_files = [
+        affected_file.replace("-affected", "") for affected_file in affected_files
+    ]
+
+    filtered_suse_oval_files = [
+        gz_file for gz_file in suse_oval_files if gz_file not in trimmed_affected_files
+    ]
+
+    return filtered_suse_oval_files
diff --git a/vulnerabilities/lib_oval.py b/vulnerabilities/lib_oval.py
@@ -62,8 +62,8 @@
 
 Available exceptions:
     - None at this time
-    
-    
+
+
 :Usage:
 
 1. Create an OvalDocument:
@@ -80,21 +80,21 @@
 
 3. Read an XML file with a single OVAL Definition (error checking omitted for brevity):
 
-    >>> tree = ElementTree()    
+    >>> tree = ElementTree()
     >>> tree.parse('test-definition.xml')
-    >>> root = tree.getroot()    
+    >>> root = tree.getroot()
     >>> definition = lib_oval.OvalDefinition(root)
-    
+
 4. Change information in the definition from #3 and write the changes
 
     >>> meta = definition.getMetadata()
     >>> repo = meta.getOvalRepositoryInformation()
     >>> repo.setMinimumSchemaVersion("5.9")
     >>> tree.write("outfilename.xml", UTF-8", True)
-        
 
 
-  
+
+
 
 TODO:
     - Add exceptions that give more detail about why a value of None is sometimes returned
@@ -253,7 +253,6 @@ def writeToFile(self, filename):
             return False
 
     def to_string(self):
-
         if not self.tree:
             return None
 
@@ -767,6 +766,12 @@ def setVersion(self, version):
         self.element.set("version", version)
         return True
 
+    def __lt__(self, other):
+        try:
+            return int(self.element.get("version")) < int(other.element.get("version"))
+        except:
+            return NotImplemented
+
     def incrementVersion(self):
         version = self.getVersion()
         if not version:

diff --git a/vulnerabilities/oval_parser.py b/vulnerabilities/oval_parser.py
@@ -25,7 +25,6 @@
 
 class OvalParser:
     def __init__(self, translations: Dict, oval_document: ET.ElementTree):
-
         self.translations = translations
         self.oval_document = OvalDocument(oval_document)
         self.all_definitions = self.oval_document.getDefinitions()
@@ -37,17 +36,14 @@ def get_data(self) -> List[Dict]:
         """
         oval_data = []
         for definition in self.all_definitions:
-
             matching_tests = self.get_tests_of_definition(definition)
             if not matching_tests:
                 continue
             definition_data = {"test_data": []}
             # TODO:this could use some data cleaning
             definition_data["description"] = definition.getMetadata().getDescription() or ""
-
             definition_data["vuln_id"] = self.get_vuln_id_from_definition(definition)
             definition_data["reference_urls"] = self.get_urls_from_definition(definition)
-
             definition_data["severity"] = self.get_severity_from_definition(definition)
 
             for test in matching_tests:
@@ -72,24 +68,30 @@ def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]:
         criteria_refs = []
 
         for child in definition.element.iter():
-
             if "test_ref" in child.attrib:
                 criteria_refs.append(child.get("test_ref"))
 
         matching_tests = []
         for ref in criteria_refs:
             oval_test = self.oval_document.getElementByID(ref)
+            # All matches will be `rpminfo_test` elements inside the `tests` element.
+            # Test for len == 2 because this IDs a pair of nested `object` and `state` elements.
             if len(oval_test.element) == 2:
                 _, state = self.get_object_state_of_test(oval_test)
                 valid_test = True
                 for child in state.element:
                     if child.get("operation") not in self.translations:
                         valid_test = False
-                        break
-                if valid_test:
-                    matching_tests.append(self.oval_document.getElementByID(ref))
+                        continue
+                    elif (
+                        child.get("operation") in self.translations
+                        # "debian_evr_string" is used in both Debian and Ubuntu test XML files; SUSE OVAL uses "evr_string".
+                        # See also https://github.com/OVALProject/Language/blob/master/docs/oval-common-schema.md
+                        and child.get("datatype") in ["evr_string", "debian_evr_string"]
+                    ):
+                        matching_tests.append(self.oval_document.getElementByID(ref))
 
-        return matching_tests
+        return sorted(set(matching_tests))
 
     def get_object_state_of_test(self, test: OvalTest) -> Tuple[OvalObject, OvalState]:
         """
@@ -109,6 +111,7 @@ def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]:
         pkg_list = []
 
         for var in obj.element:
+            # It appears that `var_ref` is used in Ubuntu OVAL but not Debian or SUSE.
             if var.get("var_ref"):
                 var_elem = self.oval_document.getElementByID(var.get("var_ref"))
                 comment = var_elem.element.get("comment")
@@ -178,9 +181,18 @@ def get_severity_from_definition(definition: OvalDefinition) -> Set[str]:
 
     @staticmethod
     def get_vuln_id_from_definition(definition):
-        # SUSE and Ubuntu OVAL files will get cves via this loop
+        # SUSE and Ubuntu OVAL files will get CVEs via this loop.
+        cve_list = []
         for child in definition.element.iter():
-            if child.get("ref_id"):
-                return child.get("ref_id")
-        # Debian OVAL files will get cves via this
-        return definition.getMetadata().getTitle()
+            if child.get("ref_id") and child.get("source"):
+                if child.get("source") == "CVE":
+                    if not child.get("ref_id").startswith("CVE"):
+                        unwanted_prefix = child.get("ref_id").split("CVE")[0]
+                        cve_list.append(child.get("ref_id").replace(unwanted_prefix, ""))
+                    else:
+                        cve_list.append(child.get("ref_id"))
+        # Debian OVAL files (no "ref_id") will get CVEs via this.
+        if len(cve_list) == 0:
+            cve_list.append(definition.getMetadata().getTitle())
+
+        return cve_list
diff --git a/vulnerabilities/tests/test_data/suse_oval/mock-definitions-only.xml b/vulnerabilities/tests/test_data/suse_oval/mock-definitions-only.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<oval_definitions xsi:schemaLocation="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux linux-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#unix unix-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-common-5 oval-common-schema.xsd"
+    xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xmlns:oval="http://oval.mitre.org/XMLSchema/oval-common-5"
+    xmlns:oval-def="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+    <definitions>
+        <definition id="oval:org.opensuse.security:def:2009030400" version="1" class="patch">
+            <metadata>
+                <title>CVE-2008-5679</title>
+                <affected family="unix">
+                </affected>
+                <reference ref_id="CVE-2008-5679" ref_url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2008-5679" source="CVE"/>
+                <description>
+	The HTML parsing engine in Opera before 9.63 allows remote attackers to execute arbitrary code via crafted web pages that trigger an invalid pointer calculation and heap corruption.
+                </description>
+            </metadata>
+        </definition>
+        <definition id="oval:org.opensuse.security:def:2009030400" version="1" class="patch">
+            <metadata>
+                <title>foobar-CVE-1234-5678</title>
+                <affected family="unix">
+                </affected>
+                <reference ref_id="foobar-CVE-1234-5678" ref_url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=foobar-CVE-1234-5678" source="CVE"/>
+                <description>
+            Blah blah blah.
+                </description>
+            </metadata>
+        </definition>
+        <definition id="oval:org.opensuse.security:def:2009030400" version="1" class="patch">
+            <metadata>
+                <title>nonesuchCVE-1111-2222</title>
+                <affected family="unix">
+                </affected>
+                <reference ref_id="nonesuchCVE-1111-2222" ref_url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=nonesuchCVE-1111-2222" source="CVE"/>
+                <description>
+                    Blah blah blah.
+                </description>
+            </metadata>
+        </definition>
+    </definitions>
+</oval_definitions>