From ee1f0cfcc6e4e04f15b4e4287a671f2fbc420f0f Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 20 May 2022 14:25:35 +0530 Subject: [PATCH 1/2] Fix pysec importer Signed-off-by: Tushar Goel --- vulnerabilities/importers/pysec.py | 58 ++-- .../test_data/pysec/pysec-expected-1.json | 55 ++++ .../test_data/pysec/pysec-expected-2.json | 44 +++ .../tests/test_data/pysec/pysec_test.json | 280 ---------------- .../tests/test_data/pysec/pysec_test_1.json | 144 +++++++++ .../tests/test_data/pysec/pysec_test_2.json | 134 ++++++++ vulnerabilities/tests/test_pysec.py | 303 ++++-------------- 7 files changed, 464 insertions(+), 554 deletions(-) create mode 100644 vulnerabilities/tests/test_data/pysec/pysec-expected-1.json create mode 100644 vulnerabilities/tests/test_data/pysec/pysec-expected-2.json delete mode 100644 vulnerabilities/tests/test_data/pysec/pysec_test.json create mode 100644 vulnerabilities/tests/test_data/pysec/pysec_test_1.json create mode 100644 vulnerabilities/tests/test_data/pysec/pysec_test_2.json diff --git a/vulnerabilities/importers/pysec.py b/vulnerabilities/importers/pysec.py index bcd4f0bef..af4e30d0c 100644 --- a/vulnerabilities/importers/pysec.py +++ b/vulnerabilities/importers/pysec.py @@ -23,17 +23,18 @@ import logging from io import BytesIO from typing import Iterable +from typing import List from typing import Optional from zipfile import ZipFile import dateparser import requests from packageurl import PackageURL -from univers.version_range import InvalidVersionRange from univers.version_range import PypiVersionRange from univers.versions import InvalidVersion from univers.versions import PypiVersion from univers.versions import SemverVersion +from univers.versions import Version from vulnerabilities.helpers import dedupe from vulnerabilities.importer import AdvisoryData @@ -70,9 +71,18 @@ def advisory_data(self) -> Iterable[AdvisoryData]: def parse_advisory_data(raw_data: dict) -> Optional[AdvisoryData]: - raw_id = raw_data["id"] + raw_id = raw_data.get("id") or "" summary = raw_data.get("summary") or "" - aliases = get_aliases(raw_data) + details = raw_data.get("details") or "" + summary_and_details = [] + if summary: + summary_and_details.append(summary) + if details: + summary_and_details.append(details) + summary = ".".join(summary_and_details) + aliases = raw_data.get("aliases") or [] + if raw_id: + aliases.append(raw_id) date_published = get_published_date(raw_data) severity = list(get_severities(raw_data)) references = get_references(raw_data, severity) @@ -80,7 +90,13 @@ def parse_advisory_data(raw_data: dict) -> Optional[AdvisoryData]: affected_packages = [] if "affected" not in raw_data: logger.error(f"affected_packages not found - {raw_id !r}") - return + return AdvisoryData( + aliases=aliases, + summary=summary, + references=references, + affected_packages=[], + date_published=date_published, + ) for affected_pkg in raw_data.get("affected") or []: purl = get_affected_purl(affected_pkg, raw_id) @@ -110,7 +126,7 @@ def parse_advisory_data(raw_data: dict) -> Optional[AdvisoryData]: ) -def fixed_filter(fixed_range) -> []: +def fixed_filter(fixed_range) -> Iterable[str]: """ Return a list of fixed version strings given a ``fixed_range`` mapping of OSV data. >>> list(fixed_filter({"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "1.6.0"}]})) @@ -124,30 +140,12 @@ def fixed_filter(fixed_range) -> []: yield fixed -def get_aliases(raw_data) -> []: - """ - aliases field is optional , id is required and these are all aliases from our perspective - converting list of two fields to a dict then , convert it to a list to make sure a list is unique - >>> get_aliases({"id": "GHSA-j3f7-7rmc-6wqj"}) - ['GHSA-j3f7-7rmc-6wqj'] - >>> get_aliases({"aliases": ["CVE-2021-40831"]}) - ['CVE-2021-40831'] - >>> get_aliases({"aliases": ["CVE-2022-22817", "GHSA-8vj2-vxx3-667w"], "id": "GHSA-j3f7-7rmc-6wqj"}) - ['CVE-2022-22817', 'GHSA-8vj2-vxx3-667w', 'GHSA-j3f7-7rmc-6wqj'] - """ - vulnerability_id = raw_data.get("id") - vulnerability_aliases = raw_data.get("aliases") or [] - if vulnerability_id: - vulnerability_aliases.append(vulnerability_id) - return vulnerability_aliases - - def get_published_date(raw_data): published = raw_data.get("published") return published and dateparser.parse(published) -def get_severities(raw_data) -> []: +def get_severities(raw_data) -> Iterable[VulnerabilitySeverity]: for sever_list in raw_data.get("severity") or []: if sever_list.get("type") == "CVSS_V3": yield VulnerabilitySeverity( @@ -173,7 +171,7 @@ def get_severities(raw_data) -> []: ) -def get_references(raw_data, severities) -> []: +def get_references(raw_data, severities) -> List[Reference]: references = raw_data.get("references") or [] return [Reference(url=ref["url"], severities=severities) for ref in references if ref] @@ -199,14 +197,16 @@ def get_affected_version_range(affected_pkg, raw_id): affected_versions = affected_pkg.get("versions") if affected_versions: try: - return PypiVersionRange(affected_versions) - except InvalidVersionRange: - logger.error(f"InvalidVersionRange affected_pkg_version_range Error - {raw_id !r} ") + return PypiVersionRange.from_versions(affected_versions) + except Exception as e: + logger.error( + f"InvalidVersionRange affected_pkg_version_range Error - {raw_id !r} {e!r}" + ) else: logger.error(f"affected_pkg_version_range not found - {raw_id !r} ") -def get_fixed_version(fixed_range, raw_id) -> []: +def get_fixed_version(fixed_range, raw_id) -> List[Version]: """ Return a list of fixed versions, using fixed_filter we get the list of fixed version strings, then we pass every element to their univers.versions , then we dedupe the result diff --git a/vulnerabilities/tests/test_data/pysec/pysec-expected-1.json b/vulnerabilities/tests/test_data/pysec/pysec-expected-1.json new file mode 100644 index 000000000..0940b214d --- /dev/null +++ b/vulnerabilities/tests/test_data/pysec/pysec-expected-1.json @@ -0,0 +1,55 @@ +{ + "aliases": [ + "CVE-2022-22817", + "GHSA-8vj2-vxx3-667w", + "PYSEC-2022-10" + ], + "summary": "PIL.ImageMath.eval in Pillow before 9.0.0 allows evaluation of arbitrary expressions, such as ones that use the Python exec method.", + "affected_packages": [ + { + "package": { + "type": "pypi", + "namespace": null, + "name": "pillow", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:pypi/1.0|1.1|1.2|1.3|1.4|1.5|1.6|1.7.0|1.7.1|1.7.2|1.7.3|1.7.4|1.7.5|1.7.6|1.7.7|1.7.8|2.0.0|2.1.0|2.2.0|2.2.1|2.2.2|2.3.0|2.3.1|2.3.2|2.4.0|2.5.0|2.5.1|2.5.2|2.5.3|2.6.0|2.6.1|2.6.2|2.7.0|2.8.0|2.8.1|2.8.2|2.9.0|3.0.0|3.1.0rc1|3.1.0rc1|3.1.0|3.1.1|3.1.2|3.2.0|3.3.0|3.3.1|3.3.2|3.3.3|3.4.0|3.4.1|3.4.2|4.0.0|4.1.0|4.1.1|4.2.0|4.2.1|4.3.0|5.0.0|5.1.0|5.2.0|5.3.0|5.4.0.dev0|5.4.0|5.4.1|6.0.0|6.1.0|6.2.0|6.2.1|6.2.2|7.0.0|7.1.0|7.1.1|7.1.2|7.2.0|8.0.0|8.0.1|8.1.0|8.1.1|8.1.2|8.2.0|8.3.0|8.3.1|8.3.2|8.4.0", + "fixed_version": "9.0.0" + } + ], + "references": [ + { + "reference_id": "", + "url": "https://pillow.readthedocs.io/en/stable/releasenotes/9.0.0.html#restrict-builtins-available-to-imagemath-eval", + "severities": [ + { + "system": "generic_textual", + "value": "HIGH" + } + ] + }, + { + "reference_id": "", + "url": "https://lists.debian.org/debian-lts-announce/2022/01/msg00018.html", + "severities": [ + { + "system": "generic_textual", + "value": "HIGH" + } + ] + }, + { + "reference_id": "", + "url": "https://github.com/advisories/GHSA-8vj2-vxx3-667w", + "severities": [ + { + "system": "generic_textual", + "value": "HIGH" + } + ] + } + ], + "date_published": "2022-01-10T14:12:00.853348+00:00" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/pysec/pysec-expected-2.json b/vulnerabilities/tests/test_data/pysec/pysec-expected-2.json new file mode 100644 index 000000000..93fcabd4f --- /dev/null +++ b/vulnerabilities/tests/test_data/pysec/pysec-expected-2.json @@ -0,0 +1,44 @@ +{ + "aliases": [ + "CVE-2021-40831", + "GHSA-j3f7-7rmc-6wqj" + ], + "summary": "Improper certificate management in AWS IoT Device SDK v2.The AWS IoT Device SDK v2 for Java, Python, C++ and Node.js appends a user supplied Certificate Authority (CA) to the root CAs instead of overriding it on macOS systems. Additionally, SNI validation is also not enabled when the CA has been \u201coverridden\u201d. TLS handshakes will thus succeed if the peer can be verified either from the user-supplied CA or the system\u2019s default trust-store. Attackers with access to a host\u2019s trust stores or are able to compromise a certificate authority already in the host's trust store (note: the attacker must also be able to spoof DNS in this case) may be able to use this issue to bypass CA pinning. An attacker could then spoof the MQTT broker, and either drop traffic and/or respond with the attacker's data, but they would not be able to forward this data on to the MQTT broker because the attacker would still need the user's private keys to authenticate against the MQTT broker. The 'aws_tls_ctx_options_override_default_trust_store_*' function within the aws-c-io submodule has been updated to address this behavior. This issue affects: Amazon Web Services AWS IoT Device SDK v2 for Java versions prior to 1.5.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Python versions prior to 1.7.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for C++ versions prior to 1.14.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Node.js versions prior to 1.6.0 on macOS. Amazon Web Services AWS-C-IO 0.10.7 on macOS.", + "affected_packages": [ + { + "package": { + "type": "pypi", + "namespace": null, + "name": "awsiotsdk", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:pypi/0.2.4|0.2.9|0.3.0|1.0.2|1.0.3|1.0.5|1.0.6|1.1.0|1.2.0|1.2.1|1.3.0|1.3.1|1.3.2|1.4.0|1.5.0|1.5.1|1.5.2|1.5.3|1.5.4|1.5.5|1.5.6|1.5.7|1.5.8|1.5.10|1.5.11|1.5.12|1.5.13|1.5.14|1.5.15|1.5.16|1.5.17|1.5.18|1.6.0|1.6.1|1.6.2", + "fixed_version": "1.7.0" + } + ], + "references": [ + { + "reference_id": "", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-40831", + "severities": [ + { + "system": "cvssv3.1_vector", + "value": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N" + } + ] + }, + { + "reference_id": "", + "url": "https://github.com/aws/aws-iot-device-sdk-cpp-v2", + "severities": [ + { + "system": "cvssv3.1_vector", + "value": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N" + } + ] + } + ], + "date_published": "2021-11-24T20:35:03+00:00" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/pysec/pysec_test.json b/vulnerabilities/tests/test_data/pysec/pysec_test.json deleted file mode 100644 index 7cebf1fcf..000000000 --- a/vulnerabilities/tests/test_data/pysec/pysec_test.json +++ /dev/null @@ -1,280 +0,0 @@ -[ - { - "id": "GHSA-j3f7-7rmc-6wqj", - "summary": "Improper certificate management in AWS IoT Device SDK v2", - "details": "The AWS IoT Device SDK v2 for Java, Python, C++ and Node.js appends a user supplied Certificate Authority (CA) to the root CAs instead of overriding it on macOS systems. Additionally, SNI validation is also not enabled when the CA has been \u201coverridden\u201d. TLS handshakes will thus succeed if the peer can be verified either from the user-supplied CA or the system\u2019s default trust-store. Attackers with access to a host\u2019s trust stores or are able to compromise a certificate authority already in the host's trust store (note: the attacker must also be able to spoof DNS in this case) may be able to use this issue to bypass CA pinning. An attacker could then spoof the MQTT broker, and either drop traffic and/or respond with the attacker's data, but they would not be able to forward this data on to the MQTT broker because the attacker would still need the user's private keys to authenticate against the MQTT broker. The 'aws_tls_ctx_options_override_default_trust_store_*' function within the aws-c-io submodule has been updated to address this behavior. This issue affects: Amazon Web Services AWS IoT Device SDK v2 for Java versions prior to 1.5.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Python versions prior to 1.7.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for C++ versions prior to 1.14.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Node.js versions prior to 1.6.0 on macOS. Amazon Web Services AWS-C-IO 0.10.7 on macOS.", - "aliases": [ - "CVE-2021-40831" - ], - "modified": "2022-02-15T05:31:22.788787Z", - "published": "2021-11-24T20:35:03Z", - "references": [ - { - "type": "ADVISORY", - "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-40831" - }, - { - "type": "WEB", - "url": "https://github.com/aws/aws-iot-device-sdk-cpp-v2" - } - ], - "affected": [ - { - "package": { - "name": "awsiotsdk", - "ecosystem": "PyPI", - "purl": "pkg:pypi/awsiotsdk" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.7.0" - } - ] - } - ], - "versions": [ - "0.2.4", - "0.2.9", - "0.3.0", - "1.0.2", - "1.0.3", - "1.0.5", - "1.0.6", - "1.1.0", - "1.2.0", - "1.2.1", - "1.3.0", - "1.3.1", - "1.3.2", - "1.4.0", - "1.5.0", - "1.5.1", - "1.5.10", - "1.5.11", - "1.5.12", - "1.5.13", - "1.5.14", - "1.5.15", - "1.5.16", - "1.5.17", - "1.5.18", - "1.5.2", - "1.5.3", - "1.5.4", - "1.5.5", - "1.5.6", - "1.5.7", - "1.5.8", - "1.6.0", - "1.6.1", - "1.6.2" - ], - "database_specific": { - "ghsa": "https://github.com/advisories/GHSA-j3f7-7rmc-6wqj", - "cvss": { - "vectorString": "CVSS:3.1/AV:A/AC:H/PR:H/UI:R/S:U/C:H/I:H/A:H", - "score": 6.3 - }, - "cwes": [ - { - "description": "The software does not validate, or incorrectly validates, a certificate.", - "cweId": "CWE-295", - "name": "Improper Certificate Validation" - } - ], - "source": "https://storage.googleapis.com/ghsa-osv/GHSA-j3f7-7rmc-6wqj.json" - } - }, - { - "package": { - "name": "aws-iot-device-sdk-v2", - "ecosystem": "npm", - "purl": "pkg:npm/aws-iot-device-sdk-v2" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.6.0" - } - ] - } - ], - "database_specific": { - "source": "https://storage.googleapis.com/ghsa-osv/GHSA-j3f7-7rmc-6wqj.json", - "ghsa": "https://github.com/advisories/GHSA-j3f7-7rmc-6wqj", - "cvss": { - "vectorString": "CVSS:3.1/AV:A/AC:H/PR:H/UI:R/S:U/C:H/I:H/A:H", - "score": 6.3 - }, - "cwes": [ - { - "description": "The software does not validate, or incorrectly validates, a certificate.", - "cweId": "CWE-295", - "name": "Improper Certificate Validation" - } - ] - } - } - ], - "schema_version": "1.2.0", - "severity": [ - {"type": "CVSS_V3", "score": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N"} - ] - - }, - { - "id": "PYSEC-2022-10", - "details": "PIL.ImageMath.eval in Pillow before 9.0.0 allows evaluation of arbitrary expressions, such as ones that use the Python exec method.", - "aliases": [ - "CVE-2022-22817", - "GHSA-8vj2-vxx3-667w" - ], - "modified": "2022-01-24T23:48:19.853348Z", - "published": "2022-01-10T14:12:00.853348Z", - "references": [ - { - "type": "WEB", - "url": "https://pillow.readthedocs.io/en/stable/releasenotes/9.0.0.html#restrict-builtins-available-to-imagemath-eval" - }, - { - "type": "WEB", - "url": "https://lists.debian.org/debian-lts-announce/2022/01/msg00018.html" - }, - { - "type": "ADVISORY", - "url": "https://github.com/advisories/GHSA-8vj2-vxx3-667w" - } - ], - "affected": [ - { - "package": { - "name": "pillow", - "ecosystem": "PyPI", - "purl": "pkg:pypi/pillow" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "9.0.0" - } - ] - } - ], - "versions": [ - "1.0", - "1.1", - "1.2", - "1.3", - "1.4", - "1.5", - "1.6", - "1.7.0", - "1.7.1", - "1.7.2", - "1.7.3", - "1.7.4", - "1.7.5", - "1.7.6", - "1.7.7", - "1.7.8", - "2.0.0", - "2.1.0", - "2.2.0", - "2.2.1", - "2.2.2", - "2.3.0", - "2.3.1", - "2.3.2", - "2.4.0", - "2.5.0", - "2.5.1", - "2.5.2", - "2.5.3", - "2.6.0", - "2.6.1", - "2.6.2", - "2.7.0", - "2.8.0", - "2.8.1", - "2.8.2", - "2.9.0", - "3.0.0", - "3.1.0", - "3.1.0.rc1", - "3.1.0rc1", - "3.1.1", - "3.1.2", - "3.2.0", - "3.3.0", - "3.3.1", - "3.3.2", - "3.3.3", - "3.4.0", - "3.4.1", - "3.4.2", - "4.0.0", - "4.1.0", - "4.1.1", - "4.2.0", - "4.2.1", - "4.3.0", - "5.0.0", - "5.1.0", - "5.2.0", - "5.3.0", - "5.4.0", - "5.4.0.dev0", - "5.4.1", - "6.0.0", - "6.1.0", - "6.2.0", - "6.2.1", - "6.2.2", - "7.0.0", - "7.1.0", - "7.1.1", - "7.1.2", - "7.2.0", - "8.0.0", - "8.0.1", - "8.1.0", - "8.1.1", - "8.1.2", - "8.2.0", - "8.3.0", - "8.3.1", - "8.3.2", - "8.4.0" - ], - "database_specific": { - "source": "https://github.com/pypa/advisory-database/blob/main/vulns/pillow/PYSEC-2022-10.yaml" - } - } - ], - "schema_version": "1.2.0", - "ecosystem_specific": { - "affected_functions": [ - "eventlet.websocket.WebSocket", - "eventlet.websocket.WebSocketWSGI" - ], - "severity": "HIGH" - - } -} -] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/pysec/pysec_test_1.json b/vulnerabilities/tests/test_data/pysec/pysec_test_1.json new file mode 100644 index 000000000..68599a619 --- /dev/null +++ b/vulnerabilities/tests/test_data/pysec/pysec_test_1.json @@ -0,0 +1,144 @@ +{ + "id": "PYSEC-2022-10", + "details": "PIL.ImageMath.eval in Pillow before 9.0.0 allows evaluation of arbitrary expressions, such as ones that use the Python exec method.", + "aliases": [ + "CVE-2022-22817", + "GHSA-8vj2-vxx3-667w" + ], + "modified": "2022-01-24T23:48:19.853348Z", + "published": "2022-01-10T14:12:00.853348Z", + "references": [ + { + "type": "WEB", + "url": "https://pillow.readthedocs.io/en/stable/releasenotes/9.0.0.html#restrict-builtins-available-to-imagemath-eval" + }, + { + "type": "WEB", + "url": "https://lists.debian.org/debian-lts-announce/2022/01/msg00018.html" + }, + { + "type": "ADVISORY", + "url": "https://github.com/advisories/GHSA-8vj2-vxx3-667w" + } + ], + "affected": [ + { + "package": { + "name": "pillow", + "ecosystem": "PyPI", + "purl": "pkg:pypi/pillow" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "9.0.0" + } + ] + } + ], + "versions": [ + "1.0", + "1.1", + "1.2", + "1.3", + "1.4", + "1.5", + "1.6", + "1.7.0", + "1.7.1", + "1.7.2", + "1.7.3", + "1.7.4", + "1.7.5", + "1.7.6", + "1.7.7", + "1.7.8", + "2.0.0", + "2.1.0", + "2.2.0", + "2.2.1", + "2.2.2", + "2.3.0", + "2.3.1", + "2.3.2", + "2.4.0", + "2.5.0", + "2.5.1", + "2.5.2", + "2.5.3", + "2.6.0", + "2.6.1", + "2.6.2", + "2.7.0", + "2.8.0", + "2.8.1", + "2.8.2", + "2.9.0", + "3.0.0", + "3.1.0", + "3.1.0.rc1", + "3.1.0rc1", + "3.1.1", + "3.1.2", + "3.2.0", + "3.3.0", + "3.3.1", + "3.3.2", + "3.3.3", + "3.4.0", + "3.4.1", + "3.4.2", + "4.0.0", + "4.1.0", + "4.1.1", + "4.2.0", + "4.2.1", + "4.3.0", + "5.0.0", + "5.1.0", + "5.2.0", + "5.3.0", + "5.4.0", + "5.4.0.dev0", + "5.4.1", + "6.0.0", + "6.1.0", + "6.2.0", + "6.2.1", + "6.2.2", + "7.0.0", + "7.1.0", + "7.1.1", + "7.1.2", + "7.2.0", + "8.0.0", + "8.0.1", + "8.1.0", + "8.1.1", + "8.1.2", + "8.2.0", + "8.3.0", + "8.3.1", + "8.3.2", + "8.4.0" + ], + "database_specific": { + "source": "https://github.com/pypa/advisory-database/blob/main/vulns/pillow/PYSEC-2022-10.yaml" + } + } + ], + "schema_version": "1.2.0", + "ecosystem_specific": { + "affected_functions": [ + "eventlet.websocket.WebSocket", + "eventlet.websocket.WebSocketWSGI" + ], + "severity": "HIGH" + + } +} diff --git a/vulnerabilities/tests/test_data/pysec/pysec_test_2.json b/vulnerabilities/tests/test_data/pysec/pysec_test_2.json new file mode 100644 index 000000000..ae8d42eda --- /dev/null +++ b/vulnerabilities/tests/test_data/pysec/pysec_test_2.json @@ -0,0 +1,134 @@ +{ + "id": "GHSA-j3f7-7rmc-6wqj", + "summary": "Improper certificate management in AWS IoT Device SDK v2", + "details": "The AWS IoT Device SDK v2 for Java, Python, C++ and Node.js appends a user supplied Certificate Authority (CA) to the root CAs instead of overriding it on macOS systems. Additionally, SNI validation is also not enabled when the CA has been \u201coverridden\u201d. TLS handshakes will thus succeed if the peer can be verified either from the user-supplied CA or the system\u2019s default trust-store. Attackers with access to a host\u2019s trust stores or are able to compromise a certificate authority already in the host's trust store (note: the attacker must also be able to spoof DNS in this case) may be able to use this issue to bypass CA pinning. An attacker could then spoof the MQTT broker, and either drop traffic and/or respond with the attacker's data, but they would not be able to forward this data on to the MQTT broker because the attacker would still need the user's private keys to authenticate against the MQTT broker. The 'aws_tls_ctx_options_override_default_trust_store_*' function within the aws-c-io submodule has been updated to address this behavior. This issue affects: Amazon Web Services AWS IoT Device SDK v2 for Java versions prior to 1.5.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Python versions prior to 1.7.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for C++ versions prior to 1.14.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Node.js versions prior to 1.6.0 on macOS. Amazon Web Services AWS-C-IO 0.10.7 on macOS.", + "aliases": [ + "CVE-2021-40831" + ], + "modified": "2022-02-15T05:31:22.788787Z", + "published": "2021-11-24T20:35:03Z", + "references": [ + { + "type": "ADVISORY", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-40831" + }, + { + "type": "WEB", + "url": "https://github.com/aws/aws-iot-device-sdk-cpp-v2" + } + ], + "affected": [ + { + "package": { + "name": "awsiotsdk", + "ecosystem": "PyPI", + "purl": "pkg:pypi/awsiotsdk" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.7.0" + } + ] + } + ], + "versions": [ + "0.2.4", + "0.2.9", + "0.3.0", + "1.0.2", + "1.0.3", + "1.0.5", + "1.0.6", + "1.1.0", + "1.2.0", + "1.2.1", + "1.3.0", + "1.3.1", + "1.3.2", + "1.4.0", + "1.5.0", + "1.5.1", + "1.5.10", + "1.5.11", + "1.5.12", + "1.5.13", + "1.5.14", + "1.5.15", + "1.5.16", + "1.5.17", + "1.5.18", + "1.5.2", + "1.5.3", + "1.5.4", + "1.5.5", + "1.5.6", + "1.5.7", + "1.5.8", + "1.6.0", + "1.6.1", + "1.6.2" + ], + "database_specific": { + "ghsa": "https://github.com/advisories/GHSA-j3f7-7rmc-6wqj", + "cvss": { + "vectorString": "CVSS:3.1/AV:A/AC:H/PR:H/UI:R/S:U/C:H/I:H/A:H", + "score": 6.3 + }, + "cwes": [ + { + "description": "The software does not validate, or incorrectly validates, a certificate.", + "cweId": "CWE-295", + "name": "Improper Certificate Validation" + } + ], + "source": "https://storage.googleapis.com/ghsa-osv/GHSA-j3f7-7rmc-6wqj.json" + } + }, + { + "package": { + "name": "aws-iot-device-sdk-v2", + "ecosystem": "npm", + "purl": "pkg:npm/aws-iot-device-sdk-v2" + }, + "ranges": [ + { + "type": "SEMVER", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.6.0" + } + ] + } + ], + "database_specific": { + "source": "https://storage.googleapis.com/ghsa-osv/GHSA-j3f7-7rmc-6wqj.json", + "ghsa": "https://github.com/advisories/GHSA-j3f7-7rmc-6wqj", + "cvss": { + "vectorString": "CVSS:3.1/AV:A/AC:H/PR:H/UI:R/S:U/C:H/I:H/A:H", + "score": 6.3 + }, + "cwes": [ + { + "description": "The software does not validate, or incorrectly validates, a certificate.", + "cweId": "CWE-295", + "name": "Improper Certificate Validation" + } + ] + } + } + ], + "schema_version": "1.2.0", + "severity": [ + {"type": "CVSS_V3", "score": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N"} + ] + +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_pysec.py b/vulnerabilities/tests/test_pysec.py index 828854c30..69be572f3 100644 --- a/vulnerabilities/tests/test_pysec.py +++ b/vulnerabilities/tests/test_pysec.py @@ -24,7 +24,9 @@ import os from unittest import TestCase +import pytz from packageurl import PackageURL +from univers.version_constraint import VersionConstraint from univers.version_range import PypiVersionRange from univers.versions import PypiVersion @@ -35,7 +37,6 @@ from vulnerabilities.importers.pysec import fixed_filter from vulnerabilities.importers.pysec import get_affected_purl from vulnerabilities.importers.pysec import get_affected_version_range -from vulnerabilities.importers.pysec import get_aliases from vulnerabilities.importers.pysec import get_fixed_version from vulnerabilities.importers.pysec import get_published_date from vulnerabilities.importers.pysec import get_references @@ -43,239 +44,28 @@ from vulnerabilities.importers.pysec import parse_advisory_data from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.severity_systems import ScoringSystem +from vulnerabilities.tests import util_tests BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/pysec", "pysec_test.json") +TEST_DATA = os.path.join(BASE_DIR, "test_data/pysec") class TestPyPIImporter(TestCase): - def test_to_advisories(self): - first_aff_range = [ - "0.2.4", - "0.2.9", - "0.3.0", - "1.0.2", - "1.0.3", - "1.0.5", - "1.0.6", - "1.1.0", - "1.2.0", - "1.2.1", - "1.3.0", - "1.3.1", - "1.3.2", - "1.4.0", - "1.5.0", - "1.5.1", - "1.5.10", - "1.5.11", - "1.5.12", - "1.5.13", - "1.5.14", - "1.5.15", - "1.5.16", - "1.5.17", - "1.5.18", - "1.5.2", - "1.5.3", - "1.5.4", - "1.5.5", - "1.5.6", - "1.5.7", - "1.5.8", - "1.6.0", - "1.6.1", - "1.6.2", - ] - second_aff_range = [ - "1.0", - "1.1", - "1.2", - "1.3", - "1.4", - "1.5", - "1.6", - "1.7.0", - "1.7.1", - "1.7.2", - "1.7.3", - "1.7.4", - "1.7.5", - "1.7.6", - "1.7.7", - "1.7.8", - "2.0.0", - "2.1.0", - "2.2.0", - "2.2.1", - "2.2.2", - "2.3.0", - "2.3.1", - "2.3.2", - "2.4.0", - "2.5.0", - "2.5.1", - "2.5.2", - "2.5.3", - "2.6.0", - "2.6.1", - "2.6.2", - "2.7.0", - "2.8.0", - "2.8.1", - "2.8.2", - "2.9.0", - "3.0.0", - "3.1.0", - "3.1.0.rc1", - "3.1.0rc1", - "3.1.1", - "3.1.2", - "3.2.0", - "3.3.0", - "3.3.1", - "3.3.2", - "3.3.3", - "3.4.0", - "3.4.1", - "3.4.2", - "4.0.0", - "4.1.0", - "4.1.1", - "4.2.0", - "4.2.1", - "4.3.0", - "5.0.0", - "5.1.0", - "5.2.0", - "5.3.0", - "5.4.0", - "5.4.0.dev0", - "5.4.1", - "6.0.0", - "6.1.0", - "6.2.0", - "6.2.1", - "6.2.2", - "7.0.0", - "7.1.0", - "7.1.1", - "7.1.2", - "7.2.0", - "8.0.0", - "8.0.1", - "8.1.0", - "8.1.1", - "8.1.2", - "8.2.0", - "8.3.0", - "8.3.1", - "8.3.2", - "8.4.0", - ] - with open(TEST_DATA) as f: + def test_to_advisories_with_summary(self): + with open(os.path.join(TEST_DATA, "pysec_test_1.json")) as f: mock_response = json.load(f) + expected_file = os.path.join(TEST_DATA, f"pysec-expected-1.json") + imported_data = parse_advisory_data(mock_response) + result = imported_data.to_dict() + util_tests.check_results_against_json(result, expected_file) - expected_advisories = [ - AdvisoryData( - aliases=["CVE-2021-40831", "GHSA-j3f7-7rmc-6wqj"], - summary="Improper certificate management in AWS IoT Device SDK v2", - affected_packages=[ - AffectedPackage( - package=PackageURL.from_string("pkg:pypi/awsiotsdk"), - affected_version_range=PypiVersionRange(first_aff_range), - fixed_version=PypiVersion("1.7.0"), - ), - ], - references=[ - Reference( - url="https://nvd.nist.gov/vuln/detail/CVE-2021-40831", - severities=[ - VulnerabilitySeverity( - system=SCORING_SYSTEMS["cvssv3.1_vector"], - value="CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", - ) - ], - ), - Reference( - url="https://github.com/aws/aws-iot-device-sdk-cpp-v2", - severities=[ - VulnerabilitySeverity( - system=SCORING_SYSTEMS["cvssv3.1_vector"], - value="CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", - ) - ], - ), - ], - date_published=datetime.datetime(2021, 11, 24, 20, 35, 3, 0).replace( - tzinfo=datetime.timezone.utc - ), - ), - AdvisoryData( - aliases=["CVE-2022-22817", "GHSA-8vj2-vxx3-667w", "PYSEC-2022-10"], - summary="", - affected_packages=[ - AffectedPackage( - package=PackageURL.from_string("pkg:pypi/pillow"), - affected_version_range=PypiVersionRange(second_aff_range), - fixed_version=PypiVersion("9.0.0"), - ) - ], - references=[ - Reference( - url="https://pillow.readthedocs.io/en/stable/releasenotes/9.0.0.html#restrict-builtins" - "-available-to-imagemath-eval", - severities=[ - VulnerabilitySeverity( - system=ScoringSystem( - identifier="generic_textual", - name="Generic textual severity rating", - url="", - notes="Severity for generic scoring systems. Contains generic textual values like High, Low etc", - ), - value="HIGH", - ) - ], - ), - Reference( - url="https://lists.debian.org/debian-lts-announce/2022/01/msg00018.html", - severities=[ - VulnerabilitySeverity( - system=ScoringSystem( - identifier="generic_textual", - name="Generic textual severity rating", - url="", - notes="Severity for generic scoring systems. Contains generic textual values like High, Low etc", - ), - value="HIGH", - ) - ], - ), - Reference( - url="https://github.com/advisories/GHSA-8vj2-vxx3-667w", - severities=[ - VulnerabilitySeverity( - system=ScoringSystem( - identifier="generic_textual", - name="Generic textual severity rating", - url="", - notes="Severity for generic scoring systems. Contains generic textual values like High, Low etc", - ), - value="HIGH", - ) - ], - ), - ], - date_published=datetime.datetime(2022, 1, 10, 14, 12, 0, 853348).replace( - tzinfo=datetime.timezone.utc - ), - ), - ] - found_data = [] - for response in mock_response: - found_data.append(parse_advisory_data(response)) - - assert expected_advisories == found_data + def test_to_advisories_without_summary(self): + with open(os.path.join(TEST_DATA, "pysec_test_2.json")) as f: + mock_response = json.load(f) + expected_file = os.path.join(TEST_DATA, f"pysec-expected-2.json") + imported_data = parse_advisory_data(mock_response) + result = imported_data.to_dict() + util_tests.check_results_against_json(result, expected_file) def test_fixed_filter(self): assert list( @@ -310,23 +100,6 @@ def test_fixed_filter(self): ) ) == ["1.5.0", "9.0g0", "10.8"] - def test_get_aliases(self): - assert get_aliases({"id": "GHSA-j3f7-7rmc-6wqj"}) == ["GHSA-j3f7-7rmc-6wqj"] - assert get_aliases({"aliases": ["CVE-2021-40831"]}) == ["CVE-2021-40831"] - self.assertCountEqual( - get_aliases({"aliases": ["CVE-2021-40831"], "id": "GHSA-j3f7-7rmc-6wqj"}), - [ - "CVE-2021-40831", - "GHSA-j3f7-7rmc-6wqj", - ], - ) - self.assertCountEqual( - get_aliases( - {"aliases": ["CVE-2022-22817", "GHSA-8vj2-vxx3-667w"], "id": "GHSA-j3f7-7rmc-6wqj"} - ), - ["CVE-2022-22817", "GHSA-8vj2-vxx3-667w", "GHSA-j3f7-7rmc-6wqj"], - ) - def test_get_published_date(self): assert get_published_date( {"id": "GHSA-j3f7-7rmc-6wqj", "published": "2022-01-10T14:12:00Z"} @@ -491,7 +264,47 @@ def test_get_affected_version_range(self): ] assert get_affected_version_range( {"versions": aff_version_range}, "GHSA-j3f7-7rmc-6wqj" - ) == (PypiVersionRange(aff_version_range)) + ) == ( + PypiVersionRange( + constraints=[ + VersionConstraint(comparator="=", version=PypiVersion(string="0.2.4")), + VersionConstraint(comparator="=", version=PypiVersion(string="0.2.9")), + VersionConstraint(comparator="=", version=PypiVersion(string="0.3.0")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.0.2")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.0.3")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.0.5")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.0.6")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.1.0")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.2.0")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.2.1")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.3.0")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.3.1")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.3.2")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.4.0")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.0")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.1")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.2")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.3")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.4")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.5")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.6")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.7")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.8")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.10")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.11")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.12")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.13")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.14")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.15")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.16")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.17")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.5.18")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.6.0")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.6.1")), + VersionConstraint(comparator="=", version=PypiVersion(string="1.6.2")), + ] + ) + ) def test_get_fixed_version(self): assert get_fixed_version({}, "GHSA-j3f7-7rmc-6wqj") == [] From ad969355914bd9fd84eb7c4937810c5712e48c22 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 20 May 2022 22:10:15 +0530 Subject: [PATCH 2/2] Add build_description in utils Rename helpers.py to utils.py Signed-off-by: Tushar Goel --- vulnerabilities/importer.py | 6 +++--- vulnerabilities/importers/alpine_linux.py | 2 +- vulnerabilities/importers/apache_httpd.py | 2 +- vulnerabilities/importers/apache_kafka.py | 2 +- vulnerabilities/importers/apache_tomcat.py | 4 ++-- vulnerabilities/importers/archlinux.py | 2 +- vulnerabilities/importers/debian.py | 10 +++++----- vulnerabilities/importers/debian_oval.py | 2 +- vulnerabilities/importers/elixir_security.py | 4 ++-- vulnerabilities/importers/gentoo.py | 2 +- vulnerabilities/importers/github.py | 12 ++++++------ vulnerabilities/importers/istio.py | 4 ++-- vulnerabilities/importers/kaybee.py | 4 ++-- vulnerabilities/importers/mozilla.py | 4 ++-- vulnerabilities/importers/nginx.py | 2 +- vulnerabilities/importers/npm.py | 4 ++-- vulnerabilities/importers/nvd.py | 2 +- vulnerabilities/importers/postgresql.py | 2 +- .../importers/project_kb_msr2019.py | 4 ++-- vulnerabilities/importers/pysec.py | 14 +++++--------- vulnerabilities/importers/redhat.py | 4 ++-- vulnerabilities/importers/retiredotnet.py | 2 +- vulnerabilities/importers/ruby.py | 4 ++-- vulnerabilities/importers/rust.py | 2 +- vulnerabilities/importers/safety_db.py | 2 +- vulnerabilities/importers/suse_backports.py | 2 +- vulnerabilities/importers/suse_scores.py | 2 +- vulnerabilities/importers/ubuntu_usn.py | 4 ++-- vulnerabilities/importers/xen.py | 4 ++-- vulnerabilities/improver.py | 2 +- vulnerabilities/improvers/default.py | 2 +- vulnerabilities/package_managers.py | 6 +++--- vulnerabilities/tests/test_apache_httpd.py | 2 +- vulnerabilities/tests/test_apache_kafka.py | 2 +- vulnerabilities/tests/test_apache_tomcat.py | 2 +- .../test_data/pysec/pysec-expected-2.json | 2 +- vulnerabilities/tests/test_debian_oval.py | 2 +- vulnerabilities/tests/test_elixir_security.py | 2 +- vulnerabilities/tests/test_gentoo.py | 2 +- vulnerabilities/tests/test_github.py | 4 ++-- vulnerabilities/tests/test_helpers.py | 6 +++--- vulnerabilities/tests/test_istio.py | 2 +- vulnerabilities/tests/test_nginx.py | 2 +- vulnerabilities/tests/test_package_managers.py | 6 +++--- vulnerabilities/tests/test_postgresql.py | 2 +- vulnerabilities/tests/test_retiredotnet.py | 2 +- vulnerabilities/tests/test_ruby.py | 2 +- vulnerabilities/tests/test_rust.py | 2 +- vulnerabilities/tests/test_safety_db.py | 2 +- vulnerabilities/tests/test_suse_scores.py | 2 +- vulnerabilities/tests/test_ubuntu.py | 2 +- vulnerabilities/{helpers.py => utils.py} | 18 ++++++++++++++++++ 52 files changed, 102 insertions(+), 88 deletions(-) rename vulnerabilities/{helpers.py => utils.py} (94%) diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index f26eb6b10..af75d079f 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -43,12 +43,12 @@ from univers.version_range import VersionRange from univers.versions import Version -from vulnerabilities.helpers import classproperty -from vulnerabilities.helpers import evolve_purl -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.oval_parser import OvalParser from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.severity_systems import ScoringSystem +from vulnerabilities.utils import classproperty +from vulnerabilities.utils import evolve_purl +from vulnerabilities.utils import nearest_patched_package logger = logging.getLogger(__name__) diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 66d87599d..51504c159 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -33,7 +33,6 @@ from packageurl import PackageURL from univers.versions import AlpineLinuxVersion -from vulnerabilities.helpers import is_cve from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Importer @@ -44,6 +43,7 @@ from vulnerabilities.references import WireSharkReference from vulnerabilities.references import XsaReference from vulnerabilities.references import ZbxReference +from vulnerabilities.utils import is_cve LOGGER = logging.getLogger(__name__) BASE_URL = "https://secdb.alpinelinux.org/" diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index 0481a550f..fe02d1ac1 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -29,13 +29,13 @@ from univers.version_range import VersionRange from univers.versions import SemverVersion -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.package_managers import GitHubTagsAPI from vulnerabilities.severity_systems import APACHE_HTTPD +from vulnerabilities.utils import nearest_patched_package class ApacheHTTPDImporter(Importer): diff --git a/vulnerabilities/importers/apache_kafka.py b/vulnerabilities/importers/apache_kafka.py index 38d14662f..223d6ac42 100644 --- a/vulnerabilities/importers/apache_kafka.py +++ b/vulnerabilities/importers/apache_kafka.py @@ -28,11 +28,11 @@ from univers.version_range import VersionRange from univers.versions import MavenVersion -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.package_managers import GitHubTagsAPI +from vulnerabilities.utils import nearest_patched_package GH_PAGE_URL = "https://raw.githubusercontent.com/apache/kafka-site/asf-site/cve-list.html" ASF_PAGE_URL = "https://kafka.apache.org/cve-list" diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py index 1f9681202..cea7846f7 100644 --- a/vulnerabilities/importers/apache_tomcat.py +++ b/vulnerabilities/importers/apache_tomcat.py @@ -30,12 +30,12 @@ from univers.versions import MavenVersion from univers.versions import SemverVersion -from vulnerabilities.helpers import create_etag -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.package_managers import MavenVersionAPI +from vulnerabilities.utils import create_etag +from vulnerabilities.utils import nearest_patched_package class ApacheTomcatImporter(Importer): diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py index 174fb19d8..b05f6ba81 100644 --- a/vulnerabilities/importers/archlinux.py +++ b/vulnerabilities/importers/archlinux.py @@ -31,11 +31,11 @@ from packageurl import PackageURL from vulnerabilities import severity_systems -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.utils import nearest_patched_package class ArchlinuxImporter(Importer): diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py index 44f1c226d..3574b990b 100644 --- a/vulnerabilities/importers/debian.py +++ b/vulnerabilities/importers/debian.py @@ -33,11 +33,6 @@ from univers.version_range import DebianVersionRange from univers.versions import DebianVersion -from vulnerabilities.helpers import AffectedPackage as LegacyAffectedPackage -from vulnerabilities.helpers import dedupe -from vulnerabilities.helpers import get_affected_packages_by_patched_package -from vulnerabilities.helpers import get_item -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Importer @@ -47,6 +42,11 @@ from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory +from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage +from vulnerabilities.utils import dedupe +from vulnerabilities.utils import get_affected_packages_by_patched_package +from vulnerabilities.utils import get_item +from vulnerabilities.utils import nearest_patched_package logger = logging.getLogger(__name__) diff --git a/vulnerabilities/importers/debian_oval.py b/vulnerabilities/importers/debian_oval.py index 7e88b4e55..8d8a0192f 100644 --- a/vulnerabilities/importers/debian_oval.py +++ b/vulnerabilities/importers/debian_oval.py @@ -26,9 +26,9 @@ import requests -from vulnerabilities.helpers import create_etag from vulnerabilities.importer import OvalImporter from vulnerabilities.package_managers import DebianVersionAPI +from vulnerabilities.utils import create_etag class DebianOvalImporter(OvalImporter): diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index 5e5006a25..1586df1c6 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -26,12 +26,12 @@ from univers.version_range import VersionRange from univers.versions import SemverVersion -from vulnerabilities.helpers import load_yaml -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference from vulnerabilities.package_managers import HexVersionAPI +from vulnerabilities.utils import load_yaml +from vulnerabilities.utils import nearest_patched_package class ElixirSecurityImporter(GitImporter): diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py index f850194bd..cc02914ae 100644 --- a/vulnerabilities/importers/gentoo.py +++ b/vulnerabilities/importers/gentoo.py @@ -26,10 +26,10 @@ from packageurl import PackageURL -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference +from vulnerabilities.utils import nearest_patched_package class GentooImporter(GitImporter): diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py index f0f71553c..d8049b7de 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/importers/github.py @@ -34,12 +34,8 @@ from univers.version_range import VersionRange from univers.version_range import build_range_from_github_advisory_constraint -from vulnerabilities import helpers from vulnerabilities import severity_systems -from vulnerabilities.helpers import AffectedPackage as LegacyAffectedPackage -from vulnerabilities.helpers import get_affected_packages_by_patched_package -from vulnerabilities.helpers import get_item -from vulnerabilities.helpers import nearest_patched_package +from vulnerabilities import utils from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Importer @@ -56,6 +52,10 @@ from vulnerabilities.package_managers import PypiVersionAPI from vulnerabilities.package_managers import RubyVersionAPI from vulnerabilities.package_managers import VersionAPI +from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage +from vulnerabilities.utils import get_affected_packages_by_patched_package +from vulnerabilities.utils import get_item +from vulnerabilities.utils import nearest_patched_package logger = logging.getLogger(__name__) @@ -191,7 +191,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]: end_cursor_exp = "" while True: graphql_query = {"query": GRAPHQL_QUERY_TEMPLATE % (ecosystem, end_cursor_exp)} - response = helpers.fetch_github_graphql_query(graphql_query) + response = utils.fetch_github_graphql_query(graphql_query) page_info = get_item(response, "data", "securityVulnerabilities", "pageInfo") end_cursor = get_item(page_info, "endCursor") diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 9b628e7af..b9f5d24b0 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -30,12 +30,12 @@ from univers.version_range import VersionRange from univers.versions import SemverVersion -from vulnerabilities.helpers import nearest_patched_package -from vulnerabilities.helpers import split_markdown_front_matter from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference from vulnerabilities.package_managers import GitHubTagsAPI +from vulnerabilities.utils import nearest_patched_package +from vulnerabilities.utils import split_markdown_front_matter is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match diff --git a/vulnerabilities/importers/kaybee.py b/vulnerabilities/importers/kaybee.py index b30529b80..6d26fd58d 100644 --- a/vulnerabilities/importers/kaybee.py +++ b/vulnerabilities/importers/kaybee.py @@ -22,11 +22,11 @@ from packageurl import PackageURL -from vulnerabilities.helpers import load_yaml -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference +from vulnerabilities.utils import load_yaml +from vulnerabilities.utils import nearest_patched_package class KaybeeImporter(GitImporter): diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index 8c89b3864..6ea367bb6 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -8,12 +8,12 @@ from packageurl import PackageURL from vulnerabilities import severity_systems -from vulnerabilities.helpers import is_cve -from vulnerabilities.helpers import split_markdown_front_matter from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.utils import is_cve +from vulnerabilities.utils import split_markdown_front_matter REPOSITORY = "mozilla/foundation-security-advisories" MFSA_FILENAME_RE = re.compile(r"mfsa(\d{4}-\d{2,3})\.(md|yml)$") diff --git a/vulnerabilities/importers/nginx.py b/vulnerabilities/importers/nginx.py index b5c3ff8ea..526f57acb 100644 --- a/vulnerabilities/importers/nginx.py +++ b/vulnerabilities/importers/nginx.py @@ -32,7 +32,6 @@ from univers.version_range import NginxVersionRange from univers.versions import SemverVersion -from vulnerabilities.helpers import evolve_purl from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Importer @@ -45,6 +44,7 @@ from vulnerabilities.package_managers import GitHubTagsAPI from vulnerabilities.package_managers import PackageVersion from vulnerabilities.severity_systems import GENERIC +from vulnerabilities.utils import evolve_purl logger = logging.getLogger(__name__) diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index 03f7253ec..80c338bea 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -33,12 +33,12 @@ from univers.version_range import VersionRange from univers.versions import SemverVersion -from vulnerabilities.helpers import load_json -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference from vulnerabilities.package_managers import NpmVersionAPI +from vulnerabilities.utils import load_json +from vulnerabilities.utils import nearest_patched_package NPM_URL = "https://registry.npmjs.org{}" diff --git a/vulnerabilities/importers/nvd.py b/vulnerabilities/importers/nvd.py index 3aff27c93..9d7d78ab5 100644 --- a/vulnerabilities/importers/nvd.py +++ b/vulnerabilities/importers/nvd.py @@ -30,7 +30,6 @@ from django.db.models.query import QuerySet from vulnerabilities import severity_systems -from vulnerabilities.helpers import get_item from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference @@ -38,6 +37,7 @@ from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory +from vulnerabilities.utils import get_item class NVDImporter(Importer): diff --git a/vulnerabilities/importers/postgresql.py b/vulnerabilities/importers/postgresql.py index fac4540fa..06754c09d 100644 --- a/vulnerabilities/importers/postgresql.py +++ b/vulnerabilities/importers/postgresql.py @@ -27,11 +27,11 @@ from packageurl import PackageURL from vulnerabilities import severity_systems -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.utils import nearest_patched_package class PostgreSQLImporter(Importer): diff --git a/vulnerabilities/importers/project_kb_msr2019.py b/vulnerabilities/importers/project_kb_msr2019.py index c75e78782..b37021a68 100644 --- a/vulnerabilities/importers/project_kb_msr2019.py +++ b/vulnerabilities/importers/project_kb_msr2019.py @@ -23,11 +23,11 @@ import csv import urllib.request -from vulnerabilities.helpers import create_etag -from vulnerabilities.helpers import is_cve from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference +from vulnerabilities.utils import create_etag +from vulnerabilities.utils import is_cve # Reading CSV file from a url using `requests` is bit too complicated. # Use `urllib.request` for that purpose. diff --git a/vulnerabilities/importers/pysec.py b/vulnerabilities/importers/pysec.py index af4e30d0c..bd08c1ad0 100644 --- a/vulnerabilities/importers/pysec.py +++ b/vulnerabilities/importers/pysec.py @@ -36,13 +36,14 @@ from univers.versions import SemverVersion from univers.versions import Version -from vulnerabilities.helpers import dedupe from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.severity_systems import SCORING_SYSTEMS +from vulnerabilities.utils import build_description +from vulnerabilities.utils import dedupe logger = logging.getLogger(__name__) @@ -74,12 +75,7 @@ def parse_advisory_data(raw_data: dict) -> Optional[AdvisoryData]: raw_id = raw_data.get("id") or "" summary = raw_data.get("summary") or "" details = raw_data.get("details") or "" - summary_and_details = [] - if summary: - summary_and_details.append(summary) - if details: - summary_and_details.append(details) - summary = ".".join(summary_and_details) + summary = build_description(summary=summary, description=details) aliases = raw_data.get("aliases") or [] if raw_id: aliases.append(raw_id) @@ -202,8 +198,8 @@ def get_affected_version_range(affected_pkg, raw_id): logger.error( f"InvalidVersionRange affected_pkg_version_range Error - {raw_id !r} {e!r}" ) - else: - logger.error(f"affected_pkg_version_range not found - {raw_id !r} ") + # else: + # logger.error(f"affected_pkg_version_range not found - {raw_id !r} ") def get_fixed_version(fixed_range, raw_id) -> List[Version]: diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index c2bc52844..4e26eb8f6 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -30,14 +30,14 @@ from univers.version_range import RpmVersionRange from vulnerabilities import severity_systems -from vulnerabilities.helpers import get_item -from vulnerabilities.helpers import requests_with_5xx_retry from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.rpm_utils import rpm_to_purl +from vulnerabilities.utils import get_item +from vulnerabilities.utils import requests_with_5xx_retry logger = logging.getLogger(__name__) diff --git a/vulnerabilities/importers/retiredotnet.py b/vulnerabilities/importers/retiredotnet.py index e8f3eb7ee..05aab923e 100644 --- a/vulnerabilities/importers/retiredotnet.py +++ b/vulnerabilities/importers/retiredotnet.py @@ -27,10 +27,10 @@ from packageurl import PackageURL -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference +from vulnerabilities.utils import AffectedPackage class RetireDotnetImporter(GitImporter): diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py index da280aea5..dd7baef42 100644 --- a/vulnerabilities/importers/ruby.py +++ b/vulnerabilities/importers/ruby.py @@ -30,12 +30,12 @@ from univers.version_range import VersionRange from univers.versions import SemverVersion -from vulnerabilities.helpers import load_yaml -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference from vulnerabilities.package_managers import RubyVersionAPI +from vulnerabilities.utils import load_yaml +from vulnerabilities.utils import nearest_patched_package class RubyImporter(GitImporter): diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py index 9f15bfde3..d9557f604 100644 --- a/vulnerabilities/importers/rust.py +++ b/vulnerabilities/importers/rust.py @@ -34,11 +34,11 @@ from univers.version_range import VersionRange from univers.versions import SemverVersion -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference from vulnerabilities.package_managers import CratesVersionAPI +from vulnerabilities.utils import nearest_patched_package class RustImporter(GitImporter): diff --git a/vulnerabilities/importers/safety_db.py b/vulnerabilities/importers/safety_db.py index b31d07a22..03211064c 100755 --- a/vulnerabilities/importers/safety_db.py +++ b/vulnerabilities/importers/safety_db.py @@ -38,11 +38,11 @@ from univers.versions import InvalidVersion from univers.versions import PypiVersion -from vulnerabilities.helpers import nearest_patched_package from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.package_managers import PypiVersionAPI +from vulnerabilities.utils import nearest_patched_package logger = logging.getLogger(__name__) diff --git a/vulnerabilities/importers/suse_backports.py b/vulnerabilities/importers/suse_backports.py index 906511b2e..8ce24c0cb 100644 --- a/vulnerabilities/importers/suse_backports.py +++ b/vulnerabilities/importers/suse_backports.py @@ -25,9 +25,9 @@ from bs4 import BeautifulSoup from packageurl import PackageURL -from vulnerabilities.helpers import create_etag from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer +from vulnerabilities.utils import create_etag class SUSEBackportsImporter(Importer): diff --git a/vulnerabilities/importers/suse_scores.py b/vulnerabilities/importers/suse_scores.py index e3976b42b..a5a3bd099 100644 --- a/vulnerabilities/importers/suse_scores.py +++ b/vulnerabilities/importers/suse_scores.py @@ -21,11 +21,11 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. from vulnerabilities import severity_systems -from vulnerabilities.helpers import fetch_yaml from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.utils import fetch_yaml URL = "https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml" diff --git a/vulnerabilities/importers/ubuntu_usn.py b/vulnerabilities/importers/ubuntu_usn.py index 3e36f0fd7..c55713ae3 100644 --- a/vulnerabilities/importers/ubuntu_usn.py +++ b/vulnerabilities/importers/ubuntu_usn.py @@ -25,11 +25,11 @@ import requests -from vulnerabilities.helpers import create_etag -from vulnerabilities.helpers import is_cve from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference +from vulnerabilities.utils import create_etag +from vulnerabilities.utils import is_cve class UbuntuUSNImporter(Importer): diff --git a/vulnerabilities/importers/xen.py b/vulnerabilities/importers/xen.py index 201aa5f51..9a00f9cc6 100644 --- a/vulnerabilities/importers/xen.py +++ b/vulnerabilities/importers/xen.py @@ -24,11 +24,11 @@ import requests -from vulnerabilities.helpers import create_etag -from vulnerabilities.helpers import is_cve from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference +from vulnerabilities.utils import create_etag +from vulnerabilities.utils import is_cve class XenImporter(Importer): diff --git a/vulnerabilities/improver.py b/vulnerabilities/improver.py index 384a364e7..4591f531e 100644 --- a/vulnerabilities/improver.py +++ b/vulnerabilities/improver.py @@ -8,9 +8,9 @@ from django.db.models.query import QuerySet from packageurl import PackageURL -from vulnerabilities.helpers import classproperty from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference +from vulnerabilities.utils import classproperty logger = logging.getLogger(__name__) diff --git a/vulnerabilities/improvers/default.py b/vulnerabilities/improvers/default.py index 9fbf95127..a8ed056ad 100644 --- a/vulnerabilities/improvers/default.py +++ b/vulnerabilities/improvers/default.py @@ -28,13 +28,13 @@ from django.db.models.query import QuerySet from packageurl import PackageURL -from vulnerabilities.helpers import evolve_purl from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.improver import MAX_CONFIDENCE from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory +from vulnerabilities.utils import evolve_purl class DefaultImprover(Improver): diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py index 4f4c7f377..bf736d6fa 100644 --- a/vulnerabilities/package_managers.py +++ b/vulnerabilities/package_managers.py @@ -36,8 +36,8 @@ from dateutil import parser as dateparser from django.utils.dateparse import parse_datetime -from vulnerabilities import helpers -from vulnerabilities.helpers import get_item +from vulnerabilities import utils +from vulnerabilities.utils import get_item logger = logging.getLogger(__name__) @@ -520,7 +520,7 @@ def fetch_tag_nodes(self, pkg: str, _DUMP_TO_FILE=False) -> Iterable[PackageVers idx = 0 while True: - response = helpers.fetch_github_graphql_query(graphql_query) + response = utils.fetch_github_graphql_query(graphql_query) # this is a convenience for testing to dump results to a file if _DUMP_TO_FILE: diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py index 9d9255fc2..da524c94d 100644 --- a/vulnerabilities/tests/test_apache_httpd.py +++ b/vulnerabilities/tests/test_apache_httpd.py @@ -28,13 +28,13 @@ from univers.version_range import VersionRange from vulnerabilities import severity_systems -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.importers.apache_httpd import ApacheHTTPDImporter from vulnerabilities.package_managers import GitHubTagsAPI from vulnerabilities.package_managers import PackageVersion +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data", "apache_httpd", "CVE-1999-1199.json") diff --git a/vulnerabilities/tests/test_apache_kafka.py b/vulnerabilities/tests/test_apache_kafka.py index dfa856529..f1d0936e3 100644 --- a/vulnerabilities/tests/test_apache_kafka.py +++ b/vulnerabilities/tests/test_apache_kafka.py @@ -26,13 +26,13 @@ from packageurl import PackageURL from univers.version_range import VersionRange -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importers.apache_kafka import ApacheKafkaImporter from vulnerabilities.importers.apache_kafka import to_version_ranges from vulnerabilities.package_managers import GitHubTagsAPI from vulnerabilities.package_managers import Version +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data", "apache_kafka", "cve-list.html") diff --git a/vulnerabilities/tests/test_apache_tomcat.py b/vulnerabilities/tests/test_apache_tomcat.py index b6fcce119..c3286c528 100644 --- a/vulnerabilities/tests/test_apache_tomcat.py +++ b/vulnerabilities/tests/test_apache_tomcat.py @@ -26,12 +26,12 @@ from packageurl import PackageURL -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importers.apache_tomcat import ApacheTomcatImporter from vulnerabilities.package_managers import MavenVersionAPI from vulnerabilities.package_managers import PackageVersion +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data", "apache_tomcat", "security-9.html") diff --git a/vulnerabilities/tests/test_data/pysec/pysec-expected-2.json b/vulnerabilities/tests/test_data/pysec/pysec-expected-2.json index 93fcabd4f..278523bba 100644 --- a/vulnerabilities/tests/test_data/pysec/pysec-expected-2.json +++ b/vulnerabilities/tests/test_data/pysec/pysec-expected-2.json @@ -3,7 +3,7 @@ "CVE-2021-40831", "GHSA-j3f7-7rmc-6wqj" ], - "summary": "Improper certificate management in AWS IoT Device SDK v2.The AWS IoT Device SDK v2 for Java, Python, C++ and Node.js appends a user supplied Certificate Authority (CA) to the root CAs instead of overriding it on macOS systems. Additionally, SNI validation is also not enabled when the CA has been \u201coverridden\u201d. TLS handshakes will thus succeed if the peer can be verified either from the user-supplied CA or the system\u2019s default trust-store. Attackers with access to a host\u2019s trust stores or are able to compromise a certificate authority already in the host's trust store (note: the attacker must also be able to spoof DNS in this case) may be able to use this issue to bypass CA pinning. An attacker could then spoof the MQTT broker, and either drop traffic and/or respond with the attacker's data, but they would not be able to forward this data on to the MQTT broker because the attacker would still need the user's private keys to authenticate against the MQTT broker. The 'aws_tls_ctx_options_override_default_trust_store_*' function within the aws-c-io submodule has been updated to address this behavior. This issue affects: Amazon Web Services AWS IoT Device SDK v2 for Java versions prior to 1.5.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Python versions prior to 1.7.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for C++ versions prior to 1.14.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Node.js versions prior to 1.6.0 on macOS. Amazon Web Services AWS-C-IO 0.10.7 on macOS.", + "summary": "Improper certificate management in AWS IoT Device SDK v2\nThe AWS IoT Device SDK v2 for Java, Python, C++ and Node.js appends a user supplied Certificate Authority (CA) to the root CAs instead of overriding it on macOS systems. Additionally, SNI validation is also not enabled when the CA has been \u201coverridden\u201d. TLS handshakes will thus succeed if the peer can be verified either from the user-supplied CA or the system\u2019s default trust-store. Attackers with access to a host\u2019s trust stores or are able to compromise a certificate authority already in the host's trust store (note: the attacker must also be able to spoof DNS in this case) may be able to use this issue to bypass CA pinning. An attacker could then spoof the MQTT broker, and either drop traffic and/or respond with the attacker's data, but they would not be able to forward this data on to the MQTT broker because the attacker would still need the user's private keys to authenticate against the MQTT broker. The 'aws_tls_ctx_options_override_default_trust_store_*' function within the aws-c-io submodule has been updated to address this behavior. This issue affects: Amazon Web Services AWS IoT Device SDK v2 for Java versions prior to 1.5.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Python versions prior to 1.7.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for C++ versions prior to 1.14.0 on macOS. Amazon Web Services AWS IoT Device SDK v2 for Node.js versions prior to 1.6.0 on macOS. Amazon Web Services AWS-C-IO 0.10.7 on macOS.", "affected_packages": [ { "package": { diff --git a/vulnerabilities/tests/test_debian_oval.py b/vulnerabilities/tests/test_debian_oval.py index 5dea100e6..8270b4499 100644 --- a/vulnerabilities/tests/test_debian_oval.py +++ b/vulnerabilities/tests/test_debian_oval.py @@ -5,10 +5,10 @@ from packageurl import PackageURL -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importers.debian_oval import DebianOvalImporter from vulnerabilities.package_managers import VersionResponse +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/") diff --git a/vulnerabilities/tests/test_elixir_security.py b/vulnerabilities/tests/test_elixir_security.py index 7361e354a..487de5f75 100644 --- a/vulnerabilities/tests/test_elixir_security.py +++ b/vulnerabilities/tests/test_elixir_security.py @@ -26,12 +26,12 @@ from packageurl import PackageURL -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importers.elixir_security import ElixirSecurityImporter from vulnerabilities.package_managers import HexVersionAPI from vulnerabilities.package_managers import Version +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/vulnerabilities/tests/test_gentoo.py b/vulnerabilities/tests/test_gentoo.py index 4d5bc54a8..3be335a9a 100644 --- a/vulnerabilities/tests/test_gentoo.py +++ b/vulnerabilities/tests/test_gentoo.py @@ -28,10 +28,10 @@ from packageurl import PackageURL -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importers.gentoo import GentooImporter +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/gentoo/glsa-201709-09.xml") diff --git a/vulnerabilities/tests/test_github.py b/vulnerabilities/tests/test_github.py index b5e186b32..9907c047e 100644 --- a/vulnerabilities/tests/test_github.py +++ b/vulnerabilities/tests/test_github.py @@ -32,7 +32,6 @@ from univers.versions import RubygemsVersion from vulnerabilities import severity_systems -from vulnerabilities.helpers import GitHubTokenError from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Reference @@ -42,6 +41,7 @@ from vulnerabilities.importers.github import process_response from vulnerabilities.importers.github import resolve_version_range from vulnerabilities.package_managers import PackageVersion +from vulnerabilities.utils import GitHubTokenError BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data", "github_api") @@ -115,7 +115,7 @@ def test_github_importer_with_missing_credentials(): list(importer.advisory_data()) -@mock.patch("vulnerabilities.helpers._get_gh_response") +@mock.patch("vulnerabilities.utils._get_gh_response") def test_github_importer_with_missing_credentials_2(mock_response): mock_response.return_value = {"message": "Bad credentials"} with pytest.raises(GitHubTokenError) as e: diff --git a/vulnerabilities/tests/test_helpers.py b/vulnerabilities/tests/test_helpers.py index d110ea5e8..7335973c3 100644 --- a/vulnerabilities/tests/test_helpers.py +++ b/vulnerabilities/tests/test_helpers.py @@ -22,9 +22,9 @@ from packageurl import PackageURL -from vulnerabilities.helpers import AffectedPackage -from vulnerabilities.helpers import nearest_patched_package -from vulnerabilities.helpers import split_markdown_front_matter +from vulnerabilities.utils import AffectedPackage +from vulnerabilities.utils import nearest_patched_package +from vulnerabilities.utils import split_markdown_front_matter def test_nearest_patched_package(): diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py index 1dcfc7903..100fb9552 100644 --- a/vulnerabilities/tests/test_istio.py +++ b/vulnerabilities/tests/test_istio.py @@ -26,12 +26,12 @@ from packageurl import PackageURL -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importers.istio import IstioImporter from vulnerabilities.package_managers import GitHubTagsAPI from vulnerabilities.package_managers import Version +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/vulnerabilities/tests/test_nginx.py b/vulnerabilities/tests/test_nginx.py index d4de6eac0..c82e89e4b 100644 --- a/vulnerabilities/tests/test_nginx.py +++ b/vulnerabilities/tests/test_nginx.py @@ -209,7 +209,7 @@ def interesting_advisories(self) -> QuerySet: ) assert interesting_advisories == advisories - @mock.patch("vulnerabilities.helpers.fetch_github_graphql_query") + @mock.patch("vulnerabilities.utils.fetch_github_graphql_query") def test_NginxBasicImprover_fetch_nginx_version_from_git_tags(self, mock_fetcher): reponse_files = [ "github-nginx-nginx-0.json", diff --git a/vulnerabilities/tests/test_package_managers.py b/vulnerabilities/tests/test_package_managers.py index 67cb10b72..f8b414ade 100644 --- a/vulnerabilities/tests/test_package_managers.py +++ b/vulnerabilities/tests/test_package_managers.py @@ -345,7 +345,7 @@ def test_fetch(self, mock_response): class TestGitHubTagsAPI: - @mock.patch("vulnerabilities.helpers.fetch_github_graphql_query") + @mock.patch("vulnerabilities.utils.fetch_github_graphql_query") def test_fetch_large_repo(self, mock_fetcher): reponse_files = [ "github-torvalds-linux-0.json", @@ -366,7 +366,7 @@ def test_fetch_large_repo(self, mock_fetcher): results = list(GitHubTagsAPI().fetch("torvalds/linux")) assert len(results) == 739 - @mock.patch("vulnerabilities.helpers.fetch_github_graphql_query") + @mock.patch("vulnerabilities.utils.fetch_github_graphql_query") def test_fetch_small_repo_1(self, mock_graphql_response): with open(os.path.join(TEST_DATA, "github", "github-nexb-scancode-toolkit-0.json")) as f: mock_graphql_response.return_value = json.load(f) @@ -430,7 +430,7 @@ def test_fetch_small_repo_1(self, mock_graphql_response): ] assert results == expected - @mock.patch("vulnerabilities.helpers.fetch_github_graphql_query") + @mock.patch("vulnerabilities.utils.fetch_github_graphql_query") def test_fetch_small_repo_2(self, mock_graphql_response): with open(os.path.join(TEST_DATA, "github", "github-nexb-vulnerablecode-0.json")) as f: mock_graphql_response.return_value = json.load(f) diff --git a/vulnerabilities/tests/test_postgresql.py b/vulnerabilities/tests/test_postgresql.py index 8121f36ea..e7f04c135 100644 --- a/vulnerabilities/tests/test_postgresql.py +++ b/vulnerabilities/tests/test_postgresql.py @@ -26,11 +26,11 @@ from packageurl import PackageURL from vulnerabilities import severity_systems -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.importers.postgresql import to_advisories +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/postgresql", "advisories.html") diff --git a/vulnerabilities/tests/test_retiredotnet.py b/vulnerabilities/tests/test_retiredotnet.py index 863136928..01b82f2c2 100644 --- a/vulnerabilities/tests/test_retiredotnet.py +++ b/vulnerabilities/tests/test_retiredotnet.py @@ -26,10 +26,10 @@ from packageurl import PackageURL -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importers.retiredotnet import RetireDotnetImporter +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/vulnerabilities/tests/test_ruby.py b/vulnerabilities/tests/test_ruby.py index f765a7667..c3bd829ca 100644 --- a/vulnerabilities/tests/test_ruby.py +++ b/vulnerabilities/tests/test_ruby.py @@ -27,12 +27,12 @@ from packageurl import PackageURL -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importers.ruby import RubyImporter from vulnerabilities.package_managers import RubyVersionAPI from vulnerabilities.package_managers import VersionResponse +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data", "ruby") diff --git a/vulnerabilities/tests/test_rust.py b/vulnerabilities/tests/test_rust.py index 134cc9939..d1f231284 100644 --- a/vulnerabilities/tests/test_rust.py +++ b/vulnerabilities/tests/test_rust.py @@ -25,7 +25,6 @@ from packageurl import PackageURL from univers.version_range import VersionRange -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importers.rust import RustImporter @@ -33,6 +32,7 @@ from vulnerabilities.importers.rust import get_advisory_data from vulnerabilities.package_managers import CratesVersionAPI from vulnerabilities.package_managers import Version +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/rust") diff --git a/vulnerabilities/tests/test_safety_db.py b/vulnerabilities/tests/test_safety_db.py index 7cdf5505c..24c271a46 100644 --- a/vulnerabilities/tests/test_safety_db.py +++ b/vulnerabilities/tests/test_safety_db.py @@ -26,13 +26,13 @@ from packageurl import PackageURL -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importers.safety_db import SafetyDbImporter from vulnerabilities.importers.safety_db import categorize_versions from vulnerabilities.package_managers import LegacyPypiVersionAPI from vulnerabilities.package_managers import Version +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data", "safety_db") diff --git a/vulnerabilities/tests/test_suse_scores.py b/vulnerabilities/tests/test_suse_scores.py index 4a5752847..db67f4467 100644 --- a/vulnerabilities/tests/test_suse_scores.py +++ b/vulnerabilities/tests/test_suse_scores.py @@ -24,11 +24,11 @@ from unittest import TestCase from vulnerabilities import severity_systems -from vulnerabilities.helpers import load_yaml from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.importers.suse_scores import SUSESeverityScoreImporter +from vulnerabilities.utils import load_yaml BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/suse_scores", "suse-cvss-scores.yaml") diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index e6fd08bff..1112fd4f4 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -8,12 +8,12 @@ from packageurl import PackageURL -from vulnerabilities.helpers import AffectedPackage from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importers.ubuntu import UbuntuImporter from vulnerabilities.oval_parser import OvalParser from vulnerabilities.package_managers import VersionResponse +from vulnerabilities.utils import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/") diff --git a/vulnerabilities/helpers.py b/vulnerabilities/utils.py similarity index 94% rename from vulnerabilities/helpers.py rename to vulnerabilities/utils.py index 66cc339c8..d3a816a61 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/utils.py @@ -292,3 +292,21 @@ def get_affected_packages_by_patched_package( package.vulnerable_package ) return affected_packages_by_patched_package + + +# This code has been vendored from scancode. +# https://github.com/nexB/scancode-toolkit/blob/develop/src/packagedcode/utils.py#L111 +def build_description(summary, description): + """ + Return a description string from a summary and description + """ + summary = (summary or "").strip() + description = (description or "").strip() + + if not description: + description = summary + else: + if summary and summary not in description: + description = "\n".join([summary, description]) + + return description