Add istio importer and tests

Signed-off-by: Tushar912 <tushar.912u@gmail.com>

vulnerabilities/importer_yielder.py

@@ -244,6 +244,15 @@
             "etags": {}
         },
     },
+    {
+        'name': 'istio',
+        'license': '',
+        'last_run': None,
+        'data_source': 'IstioDataSource',
+        'data_source_cfg': {
+            'repository_url': 'https://github.com/istio/istio.io'
+        },
+    },
 
 ]
 

vulnerabilities/importers/__init__.py

@@ -45,3 +45,4 @@
 from vulnerabilities.importers.ubuntu import UbuntuDataSource
 from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource
 from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource
+from vulnerabilities.importers.istio import IstioDataSource

vulnerabilities/importers/istio.py

@@ -0,0 +1,173 @@
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import asyncio
+from typing import List, Set
+
+from dephell_specifier import RangeSpecifier
+from packageurl import PackageURL
+
+from vulnerabilities.data_source import Advisory, GitDataSource, Reference
+from vulnerabilities.package_managers import GitHubTagsAPI
+
+
+class IstioDataSource(GitDataSource):
+    def __enter__(self):
+        super(IstioDataSource, self).__enter__()
+
+        if not getattr(self, "_added_files", None):
+            self._added_files, self._updated_files = self.file_changes(
+                recursive=True, file_ext="md", subdir="./content/en/news/security"
+            )
+        self.version_api = GitHubTagsAPI()
+        self.set_api()
+
+    def set_api(self):
+        asyncio.run(self.version_api.load_api(["istio/istio"]))
+
+    def updated_advisories(self) -> Set[Advisory]:
+        files = self._updated_files
+        advisories = []
+        for f in files:
+            processed_data = self.process_file(f)
+            if processed_data:
+                advisories.extend(processed_data)
+        return self.batch_advisories(advisories)
+
+    def added_advisories(self) -> Set[Advisory]:
+        files = self._added_files
+        advisories = []
+        for f in files:
+            processed_data = self.process_file(f)
+            if processed_data:
+                advisories.extend(processed_data)
+        return self.batch_advisories(advisories)
+
+    def get_versions_for_pkg_from_range_list(self, version_range_list):
+        # Takes a list of version ranges(affected) of a package
+        # as parameter and returns a tuple of safe package versions and
+        # vulnerable package versions
+
+        safe_pkg_versions = []
+        vuln_pkg_versions = []
+        all_version_list = self.version_api.get("istio/istio")
+        if not version_range_list:
+            return all_version_list, []
+        version_ranges = {RangeSpecifier(r) for r in version_range_list}
+        for version in all_version_list:
+            if any([version in v for v in version_ranges]):
+                vuln_pkg_versions.append(version)
+
+        safe_pkg_versions = set(all_version_list) - set(vuln_pkg_versions)
+        return safe_pkg_versions, vuln_pkg_versions
+
+    def get_data_from_md(self, file):
+        data = {}
+        for line in file:
+            line = line.strip()
+            line = line.split()
+            if len(line) > 0 and line is not None:
+
+                start = line[0]
+
+                if start == "title:":
+                    data["title"] = " ".join(line[1:])
+                elif start == "description:":
+                    data["description"] = " ".join(line[1:])
+                elif start == "cves:":
+                    data["cves"] = " ".join(line[1:])
+                    data["cves"] = data["cves"].replace("[", "")
+                    data["cves"] = data["cves"].replace("]", "")
+                    data["cves"] = data["cves"].split(",")
+
+                elif start == "releases:":
+                    data["releases"] = " ".join(line[1:])
+                    data["releases"] = data["releases"].replace("[", "")
+                    data["releases"] = data["releases"].replace("]", "")
+                    data["releases"] = data["releases"].replace('"', "")
+                    data["releases"] = data["releases"].split(",")
+        releases = []
+        if data.get("releases"):
+            for release in data["releases"]:
+                release = release.strip()
+                release = release.split(" ")
+                if len(release) > 2:
+                    lbound = ">=" + release[0]
+                    ubound = "<=" + release[2]
+                    releases.append(lbound + "," + ubound)
+        data["releases"] = releases
+
+        return data
+
+    def process_file(self, path):
+
+        advisories = []
+
+        with open(path) as f:
+            data = {}
+
+            data = self.get_data_from_md(f)
+
+            if not data.get("cves"):
+                data["cves"] = [""]
+
+            for cve_id in data["cves"]:
+
+                if not cve_id.startswith("CVE"):
+                    continue
+
+                safe_pkg_versions = []
+                vuln_pkg_versions = []
+
+                if not data.get("releases"):
+                    data["releases"] = []
+
+                (
+                    safe_pkg_versions,
+                    vuln_pkg_versions,
+                ) = self.get_versions_for_pkg_from_range_list(data["releases"])
+
+                safe_purls = []
+                vuln_purls = []
+
+                cve_id = cve_id
+
+                safe_purls = {
+                    PackageURL(name="istio", type="golang", version=version)
+                    for version in safe_pkg_versions
+                }
+
+                vuln_purls = {
+                    PackageURL(name="istio", type="golang", version=version)
+                    for version in vuln_pkg_versions
+                }
+
+                advisories.append(
+                    Advisory(
+                        summary=data["description"],
+                        impacted_package_urls=vuln_purls,
+                        resolved_package_urls=safe_purls,
+                        cve_id=cve_id,
+                    )
+                )
+
+        return advisories

vulnerabilities/tests/test_data/istio/test_file.md

@@ -0,0 +1,15 @@
+---
+title: ISTIO-SECURITY-2019-001
+subtitle: Security Bulletin
+description: Incorrect access control.
+cves: [CVE-2019-12243]
+cvss: "8.9"
+vector: "CVSS:3.0/AV:A/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:N/E:H/RL:O/RC:C"
+releases: ["1.1 to 1.1.15", "1.2 to 1.2.6", "1.3 to 1.3.1"]
+publishdate: 2019-05-28
+keywords: [CVE]
+skip_seealso: true
+aliases:
+    - /blog/2019/cve-2019-12243
+    - /news/2019/cve-2019-12243
+---

vulnerabilities/tests/test_istio.py

@@ -0,0 +1,120 @@
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import os
+from collections import OrderedDict
+from unittest import TestCase
+
+from packageurl import PackageURL
+
+from vulnerabilities.data_source import Advisory, Reference
+from vulnerabilities.importers.istio import IstioDataSource
+from vulnerabilities.package_managers import GitHubTagsAPI
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class TestIstioDataSource(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        data_source_cfg = {
+            "repository_url": "",
+        }
+        cls.data_src = IstioDataSource(1, config=data_source_cfg)
+        cls.data_src.version_api = GitHubTagsAPI(
+            {
+                "istio/istio": [
+                    "1.1.0-rc.0",
+                    "1.1.0-rc.1",
+                    "1.1.0-rc.2",
+                    "1.1.0-rc.3",
+                    "1.1.0-rc.4",
+                    "1.1.0-rc.5",
+                    "1.1.0-rc.6",
+                    "1.1.0-snapshot.2",
+                    "1.1.0-snapshot.3",
+                ]
+            }
+        )
+
+    def test_process_file(self):
+
+        path = os.path.join(BASE_DIR, "test_data/istio/test_file.md")
+        expected_data = [
+            Advisory(
+                summary=("Incorrect access control."),
+                impacted_package_urls={
+                    PackageURL(
+                        type="golang",
+                        name="istio",
+                        version="1.1.0-snapshot.2",
+                    ),
+                    PackageURL(
+                        type="golang",
+                        name="istio",
+                        version="1.1.0-snapshot.3",
+                    ),
+                },
+                resolved_package_urls={
+                    PackageURL(
+                        type="golang",
+                        name="istio",
+                        version="1.1.0-rc.2",
+                    ),
+                    PackageURL(
+                        type="golang",
+                        name="istio",
+                        version="1.1.0-rc.4",
+                    ),
+                    PackageURL(
+                        type="golang",
+                        name="istio",
+                        version="1.1.0-rc.3",
+                    ),
+                    PackageURL(
+                        type="golang",
+                        name="istio",
+                        version="1.1.0-rc.0",
+                    ),
+                    PackageURL(
+                        type="golang",
+                        name="istio",
+                        version="1.1.0-rc.5",
+                    ),
+                    PackageURL(
+                        type="golang",
+                        name="istio",
+                        version="1.1.0-rc.1",
+                    ),
+                    PackageURL(
+                        type="golang",
+                        name="istio",
+                        version="1.1.0-rc.6",
+                    ),
+                },
+                cve_id="CVE-2019-12243",
+            )
+        ]
+
+        found_data = self.data_src.process_file(path)
+
+        assert expected_data == found_data