From 16d73786f1549f60b07be2ef418525862e637b4a Mon Sep 17 00:00:00 2001 From: Tushar912 Date: Fri, 29 Jan 2021 18:52:23 +0530 Subject: [PATCH] Add istio importer and tests Signed-off-by: Tushar912 --- vulnerabilities/importer_yielder.py | 9 + vulnerabilities/importers/__init__.py | 1 + vulnerabilities/importers/istio.py | 173 ++++++++++++++++++ .../tests/test_data/istio/test_file.md | 15 ++ vulnerabilities/tests/test_istio.py | 120 ++++++++++++ 5 files changed, 318 insertions(+) create mode 100644 vulnerabilities/importers/istio.py create mode 100644 vulnerabilities/tests/test_data/istio/test_file.md create mode 100644 vulnerabilities/tests/test_istio.py diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index 9400979b3..2f476c0c0 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -244,6 +244,15 @@ "etags": {} }, }, + { + 'name': 'istio', + 'license': '', + 'last_run': None, + 'data_source': 'IstioDataSource', + 'data_source_cfg': { + 'repository_url': 'https://github.com/istio/istio.io' + }, + }, ] diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 14e82a77f..71761d5e4 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -45,3 +45,4 @@ from vulnerabilities.importers.ubuntu import UbuntuDataSource from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource +from vulnerabilities.importers.istio import IstioDataSource diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py new file mode 100644 index 000000000..ce3244d09 --- /dev/null +++ b/vulnerabilities/importers/istio.py @@ -0,0 +1,173 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import asyncio +from typing import List, Set + +from dephell_specifier import RangeSpecifier +from packageurl import PackageURL + +from vulnerabilities.data_source import Advisory, GitDataSource, Reference +from vulnerabilities.package_managers import GitHubTagsAPI + + +class IstioDataSource(GitDataSource): + def __enter__(self): + super(IstioDataSource, self).__enter__() + + if not getattr(self, "_added_files", None): + self._added_files, self._updated_files = self.file_changes( + recursive=True, file_ext="md", subdir="./content/en/news/security" + ) + self.version_api = GitHubTagsAPI() + self.set_api() + + def set_api(self): + asyncio.run(self.version_api.load_api(["istio/istio"])) + + def updated_advisories(self) -> Set[Advisory]: + files = self._updated_files + advisories = [] + for f in files: + processed_data = self.process_file(f) + if processed_data: + advisories.extend(processed_data) + return self.batch_advisories(advisories) + + def added_advisories(self) -> Set[Advisory]: + files = self._added_files + advisories = [] + for f in files: + processed_data = self.process_file(f) + if processed_data: + advisories.extend(processed_data) + return self.batch_advisories(advisories) + + def get_versions_for_pkg_from_range_list(self, version_range_list): + # Takes a list of version ranges(affected) of a package + # as parameter and returns a tuple of safe package versions and + # vulnerable package versions + + safe_pkg_versions = [] + vuln_pkg_versions = [] + all_version_list = self.version_api.get("istio/istio") + if not version_range_list: + return all_version_list, [] + version_ranges = {RangeSpecifier(r) for r in version_range_list} + for version in all_version_list: + if any([version in v for v in version_ranges]): + vuln_pkg_versions.append(version) + + safe_pkg_versions = set(all_version_list) - set(vuln_pkg_versions) + return safe_pkg_versions, vuln_pkg_versions + + def get_data_from_md(self, file): + data = {} + for line in file: + line = line.strip() + line = line.split() + if len(line) > 0 and line is not None: + + start = line[0] + + if start == "title:": + data["title"] = " ".join(line[1:]) + elif start == "description:": + data["description"] = " ".join(line[1:]) + elif start == "cves:": + data["cves"] = " ".join(line[1:]) + data["cves"] = data["cves"].replace("[", "") + data["cves"] = data["cves"].replace("]", "") + data["cves"] = data["cves"].split(",") + + elif start == "releases:": + data["releases"] = " ".join(line[1:]) + data["releases"] = data["releases"].replace("[", "") + data["releases"] = data["releases"].replace("]", "") + data["releases"] = data["releases"].replace('"', "") + data["releases"] = data["releases"].split(",") + releases = [] + if data.get("releases"): + for release in data["releases"]: + release = release.strip() + release = release.split(" ") + if len(release) > 2: + lbound = ">=" + release[0] + ubound = "<=" + release[2] + releases.append(lbound + "," + ubound) + data["releases"] = releases + + return data + + def process_file(self, path): + + advisories = [] + + with open(path) as f: + data = {} + + data = self.get_data_from_md(f) + + if not data.get("cves"): + data["cves"] = [""] + + for cve_id in data["cves"]: + + if not cve_id.startswith("CVE"): + continue + + safe_pkg_versions = [] + vuln_pkg_versions = [] + + if not data.get("releases"): + data["releases"] = [] + + ( + safe_pkg_versions, + vuln_pkg_versions, + ) = self.get_versions_for_pkg_from_range_list(data["releases"]) + + safe_purls = [] + vuln_purls = [] + + cve_id = cve_id + + safe_purls = { + PackageURL(name="istio", type="golang", version=version) + for version in safe_pkg_versions + } + + vuln_purls = { + PackageURL(name="istio", type="golang", version=version) + for version in vuln_pkg_versions + } + + advisories.append( + Advisory( + summary=data["description"], + impacted_package_urls=vuln_purls, + resolved_package_urls=safe_purls, + cve_id=cve_id, + ) + ) + + return advisories diff --git a/vulnerabilities/tests/test_data/istio/test_file.md b/vulnerabilities/tests/test_data/istio/test_file.md new file mode 100644 index 000000000..f58b20d9b --- /dev/null +++ b/vulnerabilities/tests/test_data/istio/test_file.md @@ -0,0 +1,15 @@ +--- +title: ISTIO-SECURITY-2019-001 +subtitle: Security Bulletin +description: Incorrect access control. +cves: [CVE-2019-12243] +cvss: "8.9" +vector: "CVSS:3.0/AV:A/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:N/E:H/RL:O/RC:C" +releases: ["1.1 to 1.1.15", "1.2 to 1.2.6", "1.3 to 1.3.1"] +publishdate: 2019-05-28 +keywords: [CVE] +skip_seealso: true +aliases: + - /blog/2019/cve-2019-12243 + - /news/2019/cve-2019-12243 +--- diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py new file mode 100644 index 000000000..4a22c76ed --- /dev/null +++ b/vulnerabilities/tests/test_istio.py @@ -0,0 +1,120 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import os +from collections import OrderedDict +from unittest import TestCase + +from packageurl import PackageURL + +from vulnerabilities.data_source import Advisory, Reference +from vulnerabilities.importers.istio import IstioDataSource +from vulnerabilities.package_managers import GitHubTagsAPI + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class TestIstioDataSource(TestCase): + @classmethod + def setUpClass(cls): + data_source_cfg = { + "repository_url": "", + } + cls.data_src = IstioDataSource(1, config=data_source_cfg) + cls.data_src.version_api = GitHubTagsAPI( + { + "istio/istio": [ + "1.1.0-rc.0", + "1.1.0-rc.1", + "1.1.0-rc.2", + "1.1.0-rc.3", + "1.1.0-rc.4", + "1.1.0-rc.5", + "1.1.0-rc.6", + "1.1.0-snapshot.2", + "1.1.0-snapshot.3", + ] + } + ) + + def test_process_file(self): + + path = os.path.join(BASE_DIR, "test_data/istio/test_file.md") + expected_data = [ + Advisory( + summary=("Incorrect access control."), + impacted_package_urls={ + PackageURL( + type="golang", + name="istio", + version="1.1.0-snapshot.2", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-snapshot.3", + ), + }, + resolved_package_urls={ + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.2", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.4", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.3", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.0", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.5", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.1", + ), + PackageURL( + type="golang", + name="istio", + version="1.1.0-rc.6", + ), + }, + cve_id="CVE-2019-12243", + ) + ] + + found_data = self.data_src.process_file(path) + + assert expected_data == found_data