Explore OvalParser() parsing process #1079

Reference: #1079 Signed-off-by: John M. Horan <johnmhoran@gmail.com>
aboutcode-org · Jan 20, 2023 · d02f542 · d02f542
1 parent 6ecc004
commit d02f542
Show file tree

Hide file tree

Showing 6 changed files with 174 additions and 205 deletions.
diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py
@@ -455,6 +455,7 @@ def get_data_from_xml_doc(
             ]
             affected_packages = []
             for test_data in definition_data["test_data"]:
+                print("\ntest_data = {}\n".format(test_data["package_list"]))
                 for package_name in test_data["package_list"]:
                     affected_version_range = test_data["version_ranges"]
                     vrc = RANGE_CLASS_BY_SCHEMES[pkg_metadata["type"]]
@@ -474,13 +475,14 @@ def get_data_from_xml_doc(
                                 affected_version_range=affected_version_range,
                             )
                         )
+            print("affected_packages = {}".format(affected_packages))
             date_published = dateparser.parse(timestamp)
             if not date_published.tzinfo:
                 date_published = date_published.replace(tzinfo=pytz.UTC)
             yield AdvisoryData(
                 aliases=[vuln_id],
                 summary=description,
-                affected_packages=affected_packages,
+                affected_packages=sorted(affected_packages),
                 references=sorted(references),
                 date_published=date_published,
             )
diff --git a/vulnerabilities/importers/suse_oval.py b/vulnerabilities/importers/suse_oval.py
@@ -28,43 +28,20 @@ def __init__(self, *args, **kwargs):
         self.translations = {"less than": "<", "equals": "=", "greater than or equal": ">="}
 
     def _fetch(self):
-        # base_url = "https://ftp.suse.com/pub/projects/security/oval/"
         page = requests.get(self.base_url).text
         soup = BeautifulSoup(page, "lxml")
 
-        # print(
-        #     [
-        #         self.base_url + node.get("href")
-        #         for node in soup.find_all("a")
-        #         if node.get("href").endswith(".gz")
-        #     ]
-        # )
-
         suse_oval_files = [
             self.base_url + node.get("href")
             for node in soup.find_all("a")
             if node.get("href").endswith(".gz")
         ]
 
-        # for testfile in suse_oval_files:
-        #     print(testfile)
-
-        # Temporary test of .gz version of one of the .xml files we test in test_suse_oval.py:
-        # suse_oval_files = [
-        #     "https://ftp.suse.com/pub/projects/security/oval/opensuse.leap.micro.5.3.xml.gz"
-        # ]
-
-        # TODO: 2023-01-18 Wednesday 18:49:06.  For some reason, if I un-comment the code below, my print above stops working.  Why?
-
-        # for suse_file in suse_oval_files:
-        #     # print("suse_file = {}".format(suse_file))
-        #     # Do we want to log as ubuntu.py does?  If so, why does debian_oval.py not log?
-        #     response = requests.get(suse_file)
-        #     # print("\nresponse = {}\n".format(response))
+        for suse_file in suse_oval_files:
+            response = requests.get(suse_file)
 
-        #     extracted = gzip.decompress(response.content)
-        #     # print("\nextracted = {}\n".format(extracted))
-        #     yield (
-        #         {"type": "rpm", "namespace": "opensuse"},
-        #         ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))),
-        #     )
+            extracted = gzip.decompress(response.content)
+            yield (
+                {"type": "rpm", "namespace": "opensuse"},
+                ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))),
+            )
diff --git a/vulnerabilities/oval_parser.py b/vulnerabilities/oval_parser.py
@@ -36,7 +36,11 @@ def get_data(self) -> List[Dict]:
         Return a list of OvalDefinition mappings.
         """
         oval_data = []
+        # print(len(self.all_definitions))
+        print("\nlen(self.all_definitions) = {}\n".format(len(self.all_definitions)))
         for definition in self.all_definitions:
+            # print(definition)
+            # print(list(definition))
 
             matching_tests = self.get_tests_of_definition(definition)
             if not matching_tests:
@@ -49,13 +53,15 @@ def get_data(self) -> List[Dict]:
             definition_data["reference_urls"] = self.get_urls_from_definition(definition)
 
             definition_data["severity"] = self.get_severity_from_definition(definition)
-
+            print("\nlen(matching_tests) = {}\n".format(len(matching_tests)))
             for test in matching_tests:
                 test_obj, test_state = self.get_object_state_of_test(test)
                 if not test_obj or not test_state:
                     continue
                 test_data = {"package_list": []}
+                print(test_obj)
                 test_data["package_list"].extend(self.get_pkgs_from_obj(test_obj))
+                print(self.get_pkgs_from_obj(test_obj))
                 version_ranges = self.get_version_range_from_state(test_state)
                 test_data["version_ranges"] = version_ranges
                 definition_data["test_data"].append(test_data)
@@ -88,8 +94,9 @@ def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]:
                         break
                 if valid_test:
                     matching_tests.append(self.oval_document.getElementByID(ref))
+                    print(matching_tests)
 
-        return matching_tests
+        return list(set(matching_tests))
 
     def get_object_state_of_test(self, test: OvalTest) -> Tuple[OvalObject, OvalState]:
         """

diff --git a/vulnerabilities/tests/test_data/suse_oval/suse-oval-CVE-2008-5679-expected.json b/vulnerabilities/tests/test_data/suse_oval/suse-oval-CVE-2008-5679-expected.json
@@ -5,18 +5,6 @@
     ],
     "summary": "\n\tThe HTML parsing engine in Opera before 9.63 allows remote attackers to execute arbitrary code via crafted web pages that trigger an invalid pointer calculation and heap corruption.\n\t",
     "affected_packages": [
-      {
-        "package": {
-          "type": "rpm",
-          "namespace": "opensuse",
-          "name": "opera",
-          "version": null,
-          "qualifiers": null,
-          "subpath": null
-        },
-        "affected_version_range": "vers:rpm/<9.63-1.1",
-        "fixed_version": null
-      },
       {
         "package": {
           "type": "rpm",