From 7f3c168265211659fb52b4cea22cddb157d258fe Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 20 Aug 2024 15:19:18 +0300 Subject: [PATCH 01/15] Migrate ( metasploit, exploit-db, kev ) to aboutcode pipeline. Set data_source as the header for the exploit table. Squash the migration files into a single file. Add test for exploit-db , metasploit Add a missing migration file Rename resources_and_notes to notes Fix Api test Refactor metasploit , exploitdb , kev improver Rename Kev tab to exploit tab Add support for exploitdb , metasploit, kev Signed-off-by: ziadhany --- vulnerabilities/api.py | 29 ++-- .../improvers/vulnerability_kev.py | 66 ------- .../migrations/0063_exploit_delete_kev.py | 131 ++++++++++++++ vulnerabilities/models.py | 73 ++++++-- vulnerabilities/pipelines/exploitdb.py | 95 ++++++++++ vulnerabilities/pipelines/metasploit.py | 78 +++++++++ .../pipelines/vulnerability_kev.py | 69 ++++++++ .../templates/vulnerability_details.html | 162 +++++++++++++----- .../tests/pipelines/test_exploitdb.py | 38 ++++ .../test_kev.py} | 27 +-- .../tests/pipelines/test_metasploit.py | 35 ++++ vulnerabilities/tests/test_api.py | 2 + .../exploitdb_improver/files_exploits.csv | 2 + .../modules_metadata_base.json | 93 ++++++++++ 14 files changed, 744 insertions(+), 156 deletions(-) delete mode 100644 vulnerabilities/improvers/vulnerability_kev.py create mode 100644 vulnerabilities/migrations/0063_exploit_delete_kev.py create mode 100644 vulnerabilities/pipelines/exploitdb.py create mode 100644 vulnerabilities/pipelines/metasploit.py create mode 100644 vulnerabilities/pipelines/vulnerability_kev.py create mode 100644 vulnerabilities/tests/pipelines/test_exploitdb.py rename vulnerabilities/tests/{test_kev_improver.py => pipelines/test_kev.py} (50%) create mode 100644 vulnerabilities/tests/pipelines/test_metasploit.py create mode 100644 vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv create mode 100644 vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 278ed636c..0b3dc2b8e 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -27,7 +27,7 @@ from rest_framework.throttling import UserRateThrottle from vulnerabilities.models import Alias -from vulnerabilities.models import Kev +from vulnerabilities.models import Exploit from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference @@ -175,10 +175,23 @@ def to_representation(self, instance): return representation -class KEVSerializer(serializers.ModelSerializer): +class ExploitSerializer(serializers.ModelSerializer): class Meta: - model = Kev - fields = ["date_added", "description", "required_action", "due_date", "resources_and_notes"] + model = Exploit + fields = [ + "date_added", + "description", + "required_action", + "due_date", + "notes", + "known_ransomware_campaign_use", + "source_date_published", + "exploit_type", + "platform", + "source_date_updated", + "data_source", + "source_url", + ] class VulnerabilitySerializer(BaseResourceSerializer): @@ -189,7 +202,7 @@ class VulnerabilitySerializer(BaseResourceSerializer): references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") aliases = AliasSerializer(many=True, source="alias") - kev = KEVSerializer(read_only=True) + exploits = ExploitSerializer(many=True, read_only=True) weaknesses = WeaknessSerializer(many=True) severity_range_score = serializers.SerializerMethodField() @@ -199,10 +212,6 @@ def to_representation(self, instance): weaknesses = data.get("weaknesses", []) data["weaknesses"] = [weakness for weakness in weaknesses if weakness is not None] - kev = data.get("kev", None) - if not kev: - data.pop("kev") - return data def get_severity_range_score(self, instance): @@ -240,7 +249,7 @@ class Meta: "affected_packages", "references", "weaknesses", - "kev", + "exploits", "severity_range_score", ] diff --git a/vulnerabilities/improvers/vulnerability_kev.py b/vulnerabilities/improvers/vulnerability_kev.py deleted file mode 100644 index 06e6c0380..000000000 --- a/vulnerabilities/improvers/vulnerability_kev.py +++ /dev/null @@ -1,66 +0,0 @@ -import logging -from typing import Iterable - -from django.db.models import QuerySet -from sphinx.util import requests - -from vulnerabilities.improver import Improver -from vulnerabilities.improver import Inference -from vulnerabilities.models import Advisory -from vulnerabilities.models import Alias -from vulnerabilities.models import Kev - -logger = logging.getLogger(__name__) - - -class VulnerabilityKevImprover(Improver): - """ - Known Exploited Vulnerabilities Improver - """ - - @property - def interesting_advisories(self) -> QuerySet: - # TODO Modify KEV improver to iterate over the vulnerabilities alias, not the advisory - return [Advisory.objects.first()] - - def get_inferences(self, advisory_data) -> Iterable[Inference]: - """ - Fetch Kev data, iterate over it to find the vulnerability with the specified alias, and create or update - the Kev instance accordingly. - """ - - kev_url = ( - "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" - ) - response = requests.get(kev_url) - kev_data = response.json() - if response.status_code != 200: - logger.error( - f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" - ) - return [] - - for kev_vul in kev_data.get("vulnerabilities", []): - alias = Alias.objects.get_or_none(alias=kev_vul["cveID"]) - if not alias: - continue - - vul = alias.vulnerability - - if not vul: - continue - - Kev.objects.update_or_create( - vulnerability=vul, - defaults={ - "description": kev_vul["shortDescription"], - "date_added": kev_vul["dateAdded"], - "required_action": kev_vul["requiredAction"], - "due_date": kev_vul["dueDate"], - "resources_and_notes": kev_vul["notes"], - "known_ransomware_campaign_use": True - if kev_vul["knownRansomwareCampaignUse"] == "Known" - else False, - }, - ) - return [] diff --git a/vulnerabilities/migrations/0063_exploit_delete_kev.py b/vulnerabilities/migrations/0063_exploit_delete_kev.py new file mode 100644 index 000000000..00d2d60fe --- /dev/null +++ b/vulnerabilities/migrations/0063_exploit_delete_kev.py @@ -0,0 +1,131 @@ +# Generated by Django 4.1.13 on 2024-09-10 18:40 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0062_package_is_ghost"), + ] + + operations = [ + migrations.CreateModel( + name="Exploit", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "date_added", + models.DateField( + blank=True, + help_text="The date the vulnerability was added to an exploit catalog.", + null=True, + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", + null=True, + ), + ), + ( + "required_action", + models.TextField( + blank=True, + help_text="The required action to address the vulnerability, typically to apply vendor updates or apply vendor mitigations or to discontinue use.", + null=True, + ), + ), + ( + "due_date", + models.DateField( + blank=True, + help_text="The date the required action is due, which applies to all USA federal civilian executive branch (FCEB) agencies, but all organizations are strongly encouraged to execute the required action", + null=True, + ), + ), + ( + "notes", + models.TextField( + blank=True, + help_text="Additional notes and resources about the vulnerability, often a URL to vendor instructions.", + null=True, + ), + ), + ( + "known_ransomware_campaign_use", + models.BooleanField( + default=False, + help_text="Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; \n or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.", + ), + ), + ( + "source_date_published", + models.DateField( + blank=True, + help_text="The date that the exploit was published or disclosed.", + null=True, + ), + ), + ( + "exploit_type", + models.TextField( + blank=True, + help_text="The type of the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "platform", + models.TextField( + blank=True, + help_text="The platform associated with the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "source_date_updated", + models.DateField( + blank=True, + help_text="The date the exploit was updated in the original upstream data source.", + null=True, + ), + ), + ( + "data_source", + models.TextField( + blank=True, + help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", + null=True, + ), + ), + ( + "source_url", + models.URLField( + blank=True, + help_text="The URL to the exploit as provided in the original upstream data source.", + null=True, + ), + ), + ( + "vulnerability", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="exploits", + to="vulnerabilities.vulnerability", + ), + ), + ], + ), + migrations.DeleteModel( + name="Kev", + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index ada9bec54..7b46bbe50 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -1389,49 +1389,90 @@ def log_fixing(cls, package, importer, source_url, related_vulnerability): ) -class Kev(models.Model): +class Exploit(models.Model): """ - Known Exploited Vulnerabilities + A vulnerability exploit is code used to + take advantage of a security flaw for unauthorized access or malicious activity. """ - vulnerability = models.OneToOneField( + vulnerability = models.ForeignKey( Vulnerability, + related_name="exploits", on_delete=models.CASCADE, - related_name="kev", ) date_added = models.DateField( - help_text="The date the vulnerability was added to the Known Exploited Vulnerabilities" - " (KEV) catalog in the format YYYY-MM-DD.", null=True, blank=True, + help_text="The date the vulnerability was added to an exploit catalog.", ) description = models.TextField( - help_text="Description of the vulnerability in the Known Exploited Vulnerabilities" - " (KEV) catalog, usually a refinement of the original CVE description" + null=True, + blank=True, + help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", ) required_action = models.TextField( + null=True, + blank=True, help_text="The required action to address the vulnerability, typically to " - "apply vendor updates or apply vendor mitigations or to discontinue use." + "apply vendor updates or apply vendor mitigations or to discontinue use.", ) due_date = models.DateField( - help_text="The date the required action is due in the format YYYY-MM-DD," - "which applies to all USA federal civilian executive branch (FCEB) agencies," - "but all organizations are strongly encouraged to execute the required action." + null=True, + blank=True, + help_text="The date the required action is due, which applies" + " to all USA federal civilian executive branch (FCEB) agencies, " + "but all organizations are strongly encouraged to execute the required action", ) - resources_and_notes = models.TextField( + notes = models.TextField( + null=True, + blank=True, help_text="Additional notes and resources about the vulnerability," - " often a URL to vendor instructions." + " often a URL to vendor instructions.", ) known_ransomware_campaign_use = models.BooleanField( default=False, - help_text="""Known if this vulnerability is known to have been leveraged as part of a ransomware campaign; - or 'Unknown' if CISA lacks confirmation that the vulnerability has been utilized for ransomware.""", + help_text="""Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; + or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.""", + ) + + source_date_published = models.DateField( + null=True, blank=True, help_text="The date that the exploit was published or disclosed." + ) + + exploit_type = models.TextField( + null=True, + blank=True, + help_text="The type of the exploit as provided by the original upstream data source.", + ) + + platform = models.TextField( + null=True, + blank=True, + help_text="The platform associated with the exploit as provided by the original upstream data source.", + ) + + source_date_updated = models.DateField( + null=True, + blank=True, + help_text="The date the exploit was updated in the original upstream data source.", + ) + + data_source = models.TextField( + null=True, + blank=True, + help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", + ) + + source_url = models.URLField( + null=True, + blank=True, + help_text="The URL to the exploit as provided in the original upstream data source.", ) @property diff --git a/vulnerabilities/pipelines/exploitdb.py b/vulnerabilities/pipelines/exploitdb.py new file mode 100644 index 000000000..0c3bdc458 --- /dev/null +++ b/vulnerabilities/pipelines/exploitdb.py @@ -0,0 +1,95 @@ +import csv +import io +import logging + +import requests + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.pipelines import VulnerableCodePipeline + +logger = logging.getLogger(__name__) + + +class ExploitDBImproverPipeline(VulnerableCodePipeline): + """ + ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with + the specified alias, and create or update the ref and ref-type accordingly. + """ + + exploit_data = None + + license_expression = "GPL-2.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploit, + ) + + def fetch_exploits(self): + exploit_db_url = ( + "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" + ) + response = requests.get(exploit_db_url) + self.exploit_data = io.StringIO(response.text) + + def add_exploit(self): + csvreader = csv.reader(self.exploit_data) + + header = next(csvreader) + for row in csvreader: + + aliases = row[11].split(";") + + for raw_alias in aliases: + + alias = Alias.objects.get_or_none(alias=raw_alias) + if not alias: + continue + + vul = alias.vulnerability + if not vul: + continue + + self.add_exploit_references(row[11], row[16], row[1], vul) + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="Exploit-DB", + defaults={ + "date_added": row[header.index("date_added")], + "description": row[header.index("description")], + "known_ransomware_campaign_use": row[header.index("verified")], + "source_date_published": row[header.index("date_published")], + "exploit_type": row[header.index("type")], + "platform": row[header.index("platform")], + "source_date_updated": row[header.index("date_updated")], + "source_url": row[header.index("source_url")], + }, + ) + + def add_exploit_references(self, ref_id, direct_url, path, vul): + url_map = { + "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", + "direct_url": direct_url, + } + + for key, url in url_map.items(): + if url: + ref, created = VulnerabilityReference.objects.update_or_create( + url=url, + defaults={ + "reference_id": ref_id, + "reference_type": VulnerabilityReference.EXPLOIT, + }, + ) + + if created: + VulnerabilityRelatedReference.objects.get_or_create( + vulnerability=vul, + reference=ref, + ) diff --git a/vulnerabilities/pipelines/metasploit.py b/vulnerabilities/pipelines/metasploit.py new file mode 100644 index 000000000..be1829ede --- /dev/null +++ b/vulnerabilities/pipelines/metasploit.py @@ -0,0 +1,78 @@ +import logging + +import requests +import saneyaml + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class MetasploitImproverPipeline(VulnerableCodePipeline): + """ + Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + metasploit_data = {} + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" + response = requests.get(url) + if response.status_code != 200: + self.log(f"Failed to fetch the Metasploit Exploits: {url}") + return + self.metasploit_data = response.json() + + def add_exploits(self): + for _, record in self.metasploit_data.items(): + vul = None + for ref in record.get("references", []): + if ref.startswith("OSVDB") or ref.startswith("URL-"): + # ignore OSV-DB and reference exploit for metasploit + continue + + if not vul: + try: + alias = Alias.objects.get(alias=ref) + except Alias.DoesNotExist: + continue + + if not alias.vulnerability: + continue + + vul = alias.vulnerability + + if not vul: + continue + + description = record.get("description", "") + notes = record.get("notes", {}) + source_date_published = record.get("disclosure_date") + platform = record.get("platform") + + path = record.get("path") + source_url = ( + f"https://github.com/rapid7/metasploit-framework/tree/master{path}" if path else "" + ) + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="Metasploit", + defaults={ + "description": description, + "notes": saneyaml.dump(notes), + "source_date_published": source_date_published, + "platform": platform, + "source_url": source_url, + }, + ) diff --git a/vulnerabilities/pipelines/vulnerability_kev.py b/vulnerabilities/pipelines/vulnerability_kev.py new file mode 100644 index 000000000..255249472 --- /dev/null +++ b/vulnerabilities/pipelines/vulnerability_kev.py @@ -0,0 +1,69 @@ +import logging + +from sphinx.util import requests + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class VulnerabilityKevPipeline(VulnerableCodePipeline): + """ + Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + kev_data = {} + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + kev_url = ( + "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + ) + response = requests.get(kev_url) + if response.status_code != 200: + self.log( + f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" + ) + return + self.kev_data = response.json() + + def add_exploits(self): + for kev_vul in self.kev_data.get("vulnerabilities", []): + cve_id = kev_vul.get("cveID") + + if not cve_id: + continue + + alias = Alias.objects.get_or_none(alias=cve_id) + + if not alias: + continue + + vul = alias.vulnerability + + if not vul: + continue + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="KEV", + defaults={ + "description": kev_vul["shortDescription"], + "date_added": kev_vul["dateAdded"], + "required_action": kev_vul["requiredAction"], + "due_date": kev_vul["dueDate"], + "notes": kev_vul["notes"], + "known_ransomware_campaign_use": True + if kev_vul["knownRansomwareCampaignUse"] == "Known" + else False, + }, + ) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index c950adad1..aeb3eb649 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -61,11 +61,11 @@ - {% if vulnerability.kev %} -
  • + {% if vulnerability.exploits %} +
  • - Known Exploited Vulnerabilities + Exploits ({{ vulnerability.exploits.count }})
    • @@ -77,7 +77,7 @@ EPSS - +
  • @@ -439,87 +439,157 @@ {% endfor %} - {% if vulnerability.kev %} -
    -
    - Known Exploited Vulnerabilities -
    - + + +
    + {% for exploit in vulnerability.exploits.all %} +
    + + + + + - - - - - - {% if vulnerability.kev.description %} + {% if exploit.date_added %} + + + + + {% endif %} + {% if exploit.description %} - + + + {% endif %} + {% if exploit.required_action %} + + + + + {% endif %} + {% if exploit.due_date %} + + + + + {% endif %} + {% if exploit.notes %} + + + + + {% endif %} + {% if exploit.known_ransomware_campaign_use %} + + + {% endif %} - {% if vulnerability.kev.required_action %} + {% if exploit.source_date_published %} - + {% endif %} - - {% if vulnerability.kev.resources_and_notes %} + {% if exploit.exploit_type %} - + {% endif %} - - {% if vulnerability.kev.due_date %} + {% if exploit.platform %} - + {% endif %} - {% if vulnerability.kev.date_added %} + {% if exploit.source_date_updated %} - + + + {% endif %} + + {% if exploit.source_url %} + + + {% endif %} - -
    data_source {{ exploit.data_source }}
    - - Known Ransomware Campaign Use: - - {{ vulnerability.kev.get_known_ransomware_campaign_use_type }}
    + + date_added: + + {{ exploit.date_added }}
    + data-tooltip="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description"> Description: {{ vulnerability.kev.description }}{{ exploit.description }}
    + + required_action: + + {{ exploit.required_action }}
    + + due_date: + + {{ exploit.due_date }}
    + + notes: + + 
    {{ exploit.notes }}
    + + known_ransomware_campaign_use: + + {{ exploit.known_ransomware_campaign_use }}
    - Required Action: + data-tooltip="The date that the exploit was published or disclosed."> + source_date_published: {{ vulnerability.kev.required_action }}{{ exploit.source_date_published }}
    - Notes: + data-tooltip="The type of the exploit as provided by the original upstream data source."> + exploit_type: {{ vulnerability.kev.resources_and_notes }}{{ exploit.exploit_type }}
    - Due Date: + data-tooltip="The platform associated with the exploit as provided by the original upstream data source."> + platform: {{ vulnerability.kev.due_date }}{{ exploit.platform }}
    - Date Added: + data-tooltip="The date the exploit was updated in the original upstream data source."> + source_date_updated: {{ vulnerability.kev.date_added }}{{ exploit.source_date_updated }}
    + + source_url: + + {{ exploit.source_url }}
    -
    - {% endif %} + + {% empty %} + + + No exploits are available. + + + {% endfor %} + + {% for severity in severities %} {% if severity.scoring_system == 'epss' %} diff --git a/vulnerabilities/tests/pipelines/test_exploitdb.py b/vulnerabilities/tests/pipelines/test_exploitdb.py new file mode 100644 index 000000000..3f30433e5 --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_exploitdb.py @@ -0,0 +1,38 @@ +import os +from unittest import mock +from unittest.mock import Mock + +import pytest + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.exploitdb import ExploitDBImproverPipeline + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "exploitdb_improver/files_exploits.csv") + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_exploit_db_improver(mock_get): + mock_response = Mock(status_code=200) + with open(TEST_DATA, "r") as f: + mock_response.text = f.read() + mock_get.return_value = mock_response + + improver = ExploitDBImproverPipeline() + + # Run the improver when there is no matching aliases + improver.execute() + + assert Exploit.objects.count() == 0 + + v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") + v1.save() + + Alias.objects.create(alias="CVE-2009-3699", vulnerability=v1) + + # Run Exploit-DB Improver again when there are matching aliases. + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/test_kev_improver.py b/vulnerabilities/tests/pipelines/test_kev.py similarity index 50% rename from vulnerabilities/tests/test_kev_improver.py rename to vulnerabilities/tests/pipelines/test_kev.py index d0b1c981a..e8931ff1a 100644 --- a/vulnerabilities/tests/test_kev_improver.py +++ b/vulnerabilities/tests/pipelines/test_kev.py @@ -1,41 +1,32 @@ import os -from datetime import datetime from unittest import mock from unittest.mock import Mock import pytest -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.improvers.vulnerability_kev import VulnerabilityKevImprover from vulnerabilities.models import Alias -from vulnerabilities.models import Kev +from vulnerabilities.models import Exploit from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.vulnerability_kev import VulnerabilityKevPipeline from vulnerabilities.utils import load_json BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "kev_data.json") +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "kev_data.json") @pytest.mark.django_db @mock.patch("requests.get") def test_kev_improver(mock_get): - advisory_data = AdvisoryData( - aliases=["CVE-2022-21831"], - summary="Possible code injection vulnerability in Rails / Active Storage", - affected_packages=[], - references=[], - date_published=datetime.now(), - ) # to just run the improver - mock_response = Mock(status_code=200) mock_response.json.return_value = load_json(TEST_DATA) mock_get.return_value = mock_response - improver = VulnerabilityKevImprover() + improver = VulnerabilityKevPipeline() # Run the improver when there is no matching aliases - improver.get_inferences(advisory_data=advisory_data) - assert Kev.objects.count() == 0 + improver.execute() + + assert Exploit.objects.count() == 0 v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") v1.save() @@ -43,5 +34,5 @@ def test_kev_improver(mock_get): Alias.objects.create(alias="CVE-2021-38647", vulnerability=v1) # Run Kev Improver again when there are matching aliases. - improver.get_inferences(advisory_data=advisory_data) - assert Kev.objects.count() == 1 + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/pipelines/test_metasploit.py b/vulnerabilities/tests/pipelines/test_metasploit.py new file mode 100644 index 000000000..65e893cba --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_metasploit.py @@ -0,0 +1,35 @@ +import os +from unittest import mock +from unittest.mock import Mock + +import pytest + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.metasploit import MetasploitImproverPipeline +from vulnerabilities.utils import load_json + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "metasploit_improver/modules_metadata_base.json") + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_metasploit_improver(mock_get): + mock_response = Mock(status_code=200) + mock_response.json.return_value = load_json(TEST_DATA) + mock_get.return_value = mock_response + + improver = MetasploitImproverPipeline() + + # Run the improver when there is no matching aliases + improver.execute() + assert Exploit.objects.count() == 0 + + v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") + Alias.objects.create(alias="CVE-2007-4387", vulnerability=v1) + + # Run metasploit Improver again when there are matching aliases. + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 8fb50243a..18807a6d7 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -296,6 +296,7 @@ def test_api_with_single_vulnerability(self): "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", }, ], + "exploits": [], } def test_api_with_single_vulnerability_with_filters(self): @@ -341,6 +342,7 @@ def test_api_with_single_vulnerability_with_filters(self): "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", }, ], + "exploits": [], } diff --git a/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv b/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv new file mode 100644 index 000000000..a63701d8c --- /dev/null +++ b/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv @@ -0,0 +1,2 @@ +id,file,description,date_published,author,type,platform,port,date_added,date_updated,verified,codes,tags,aliases,screenshot_url,application_url,source_url +16929,exploits/aix/dos/16929.rb,"AIX Calendar Manager Service Daemon (rpc.cmsd) Opcode 21 - Buffer Overflow (Metasploit)",2010-11-11,Metasploit,dos,aix,,2010-11-11,2011-03-06,1,CVE-2009-3699;OSVDB-58726,"Metasploit Framework (MSF)",,,,http://aix.software.ibm.com/aix/efixes/security/cmsd_advisory.asc diff --git a/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json b/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json new file mode 100644 index 000000000..e9351a1df --- /dev/null +++ b/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json @@ -0,0 +1,93 @@ +{ + "auxiliary_admin/2wire/xslt_password_reset": { + "name": "2Wire Cross-Site Request Forgery Password Reset Vulnerability", + "fullname": "auxiliary/admin/2wire/xslt_password_reset", + "aliases": [ + ], + "rank": 300, + "disclosure_date": "2007-08-15", + "type": "auxiliary", + "author": [ + "hkm ", + "Travis Phillips" + ], + "description": "This module will reset the admin password on a 2Wire wireless router. This is\n done by using the /xslt page where authentication is not required, thus allowing\n configuration changes (such as resetting the password) as administrators.", + "references": [ + "CVE-2007-4387", + "OSVDB-37667", + "BID-36075", + "URL-https://seclists.org/bugtraq/2007/Aug/225" + ], + "platform": "", + "arch": "", + "rport": 80, + "autofilter_ports": [ + 80, + 8080, + 443, + 8000, + 8888, + 8880, + 8008, + 3000, + 8443 + ], + "autofilter_services": [ + "http", + "https" + ], + "targets": null, + "mod_time": "2020-10-02 17:38:06 +0000", + "path": "/modules/auxiliary/admin/2wire/xslt_password_reset.rb", + "is_install_path": true, + "ref_name": "admin/2wire/xslt_password_reset", + "check": false, + "post_auth": false, + "default_credential": false, + "notes": { + }, + "session_types": false, + "needs_cleanup": false, + "actions": [ + ] + }, + "post_firefox/manage/webcam_chat": { + "name": "Firefox Webcam Chat on Privileged Javascript Shell", + "fullname": "post/firefox/manage/webcam_chat", + "aliases": [ + + ], + "rank": 300, + "disclosure_date": "2014-05-13", + "type": "post", + "author": [ + "joev " + ], + "description": "This module allows streaming a webcam from a privileged Firefox Javascript shell.", + "references": [ + "URL-http://www.rapid7.com/db/modules/exploit/firefox/local/exec_shellcode" + ], + "platform": "", + "arch": "", + "rport": null, + "autofilter_ports": null, + "autofilter_services": null, + "targets": null, + "mod_time": "2023-02-08 13:47:34 +0000", + "path": "/modules/post/firefox/manage/webcam_chat.rb", + "is_install_path": true, + "ref_name": "firefox/manage/webcam_chat", + "check": false, + "post_auth": false, + "default_credential": false, + "notes": { + }, + "session_types": [ + + ], + "needs_cleanup": null, + "actions": [ + + ] + } +} \ No newline at end of file From c513713b4905ff2c1bbf43bf27bd34802a3042be Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 17 Sep 2024 17:48:51 +0300 Subject: [PATCH 02/15] Implement the appropriate LoopProgress progress bar. Refactor the error handling logic in the code. Signed-off-by: ziadhany --- .../pipelines/enhance_with_exploitdb.py | 149 ++++++++++++++++++ vulnerabilities/pipelines/enhance_with_kev.py | 82 ++++++++++ .../pipelines/enhance_with_metasploit.py | 103 ++++++++++++ vulnerabilities/pipelines/exploitdb.py | 95 ----------- vulnerabilities/pipelines/metasploit.py | 78 --------- .../pipelines/vulnerability_kev.py | 69 -------- .../tests/pipelines/test_exploitdb.py | 2 +- vulnerabilities/tests/pipelines/test_kev.py | 2 +- .../tests/pipelines/test_metasploit.py | 2 +- 9 files changed, 337 insertions(+), 245 deletions(-) create mode 100644 vulnerabilities/pipelines/enhance_with_exploitdb.py create mode 100644 vulnerabilities/pipelines/enhance_with_kev.py create mode 100644 vulnerabilities/pipelines/enhance_with_metasploit.py delete mode 100644 vulnerabilities/pipelines/exploitdb.py delete mode 100644 vulnerabilities/pipelines/metasploit.py delete mode 100644 vulnerabilities/pipelines/vulnerability_kev.py diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py new file mode 100644 index 000000000..70ccd31f1 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -0,0 +1,149 @@ +import csv +import io +import logging + +import requests +from aboutcode.pipeline import LoopProgress +from dateutil import parser as dateparser +from django.db import DataError + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class ExploitDBImproverPipeline(VulnerableCodePipeline): + """ + ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with + the specified alias, and create or update the ref and ref-type accordingly. + """ + + license_expression = "GPL-2.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploit, + ) + + def fetch_exploits(self): + exploit_db_url = ( + "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" + ) + + try: + response = requests.get(exploit_db_url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the Exploit-DB Exploits: {exploit_db_url} - {http_err}", + level=logging.ERROR, + ) + raise + + self.exploit_data = io.StringIO(response.text) + + def add_exploit(self): + csvreader = csv.reader(self.exploit_data) + header = next(csvreader) + + raw_data = list(csvreader) + fetched_exploit_count = len(raw_data) + + vulnerability_exploit_count = 0 + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + + for row in progress.iter(raw_data): + vulnerability_exploit_count += add_vulnerability_exploit(row, header, self.log) + + self.log( + f"Successfully added {vulnerability_exploit_count:,d} exploit-db vulnerability exploit" + ) + + +def add_vulnerability_exploit(row, header, logger): + vulnerability = None + aliases = row[11].split(";") + + for raw_alias in aliases: + try: + if alias := Alias.objects.get(alias=raw_alias): + vulnerability = alias.vulnerability + break + except Alias.DoesNotExist: + continue + + if not vulnerability: + logger(f"No vulnerability found for aliases {aliases}") + return 0 + + add_exploit_references(row[11], row[16], row[1], vulnerability, logger) + + date_added = parse_date(row[header.index("date_added")]) + source_date_published = parse_date(row[header.index("date_published")]) + source_date_updated = parse_date(row[header.index("date_updated")]) + + try: + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="Exploit-DB", + defaults={ + "date_added": date_added, + "description": row[header.index("description")], + "known_ransomware_campaign_use": row[header.index("verified")], + "source_date_published": source_date_published, + "exploit_type": row[header.index("type")], + "platform": row[header.index("platform")], + "source_date_updated": source_date_updated, + "source_url": row[header.index("source_url")], + }, + ) + except DataError as e: + logger( + f"Failed to Create the Vulnerability Exploit-DB: {e}", + level=logging.ERROR, + ) + return 1 + + +def add_exploit_references(ref_id, direct_url, path, vul, logger): + url_map = { + "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", + "direct_url": direct_url, + } + + for key, url in url_map.items(): + if url: + try: + ref, created = VulnerabilityReference.objects.update_or_create( + url=url, + defaults={ + "reference_id": ref_id, + "reference_type": VulnerabilityReference.EXPLOIT, + }, + ) + + if created: + VulnerabilityRelatedReference.objects.get_or_create( + vulnerability=vul, + reference=ref, + ) + + except DataError as e: + logger( + f"Failed to Create the Vulnerability Reference For Exploit-DB: {e}", + level=logging.ERROR, + ) + + +def parse_date(date_string): + if date_string: + try: + date_obj = dateparser.parse(date_string).date() + return date_obj.strftime("%Y-%m-%d") + except (ValueError, TypeError, Exception) as e: + logging.error(f"Error while parsing ExploitDB date '{date_string}': {e}") + return diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py new file mode 100644 index 000000000..98732a721 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -0,0 +1,82 @@ +import logging + +import requests +from aboutcode.pipeline import LoopProgress + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class VulnerabilityKevPipeline(VulnerableCodePipeline): + """ + Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + kev_url = ( + "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + ) + try: + response = requests.get(kev_url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the KEV Exploits: {kev_url} - {http_err}", + level=logging.ERROR, + ) + raise + self.kev_data = response.json() + + def add_exploits(self): + fetched_exploit_count = self.kev_data.get("count") + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") + + vulnerability_exploit_count = 0 + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + + for record in progress.iter(self.kev_data.get("vulnerabilities", [])): + vulnerability_exploit_count += add_vulnerability_exploit( + kev_vul=record, + logger=self.log, + ) + + self.log(f"Successfully added {vulnerability_exploit_count:,d} kev exploit") + + +def add_vulnerability_exploit(kev_vul, logger): + cve_id = kev_vul.get("cveID") + + vulnerability = None + try: + if alias := Alias.objects.get(alias=cve_id): + vulnerability = alias.vulnerability + except Alias.DoesNotExist: + logger(f"No vulnerability found for aliases {cve_id}") + return 0 + + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="KEV", + defaults={ + "description": kev_vul["shortDescription"], + "date_added": kev_vul["dateAdded"], + "required_action": kev_vul["requiredAction"], + "due_date": kev_vul["dueDate"], + "notes": kev_vul["notes"], + "known_ransomware_campaign_use": True + if kev_vul["knownRansomwareCampaignUse"] == "Known" + else False, + }, + ) + return 1 diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py new file mode 100644 index 000000000..a569249d6 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -0,0 +1,103 @@ +import logging + +import requests +import saneyaml +from aboutcode.pipeline import LoopProgress +from dateutil import parser as dateparser + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class MetasploitImproverPipeline(VulnerableCodePipeline): + """ + Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + license_expression = "BSD-3-clause" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_vulnerability_exploits, + ) + + def fetch_exploits(self): + url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" + self.log(f"Fetching {url}") + try: + response = requests.get(url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the Metasploit Exploits: {url} - {http_err}", level=logging.ERROR + ) + raise + + self.metasploit_data = response.json() + + def add_vulnerability_exploits(self): + fetched_exploit_count = len(self.metasploit_data) + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") + + vulnerability_exploit_count = 0 + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + for _, record in progress.iter(self.metasploit_data.items()): + vulnerability_exploit_count += add_vulnerability_exploit( + record=record, + logger=self.log, + ) + self.log(f"Successfully added {vulnerability_exploit_count:,d} vulnerability exploit") + + +def add_vulnerability_exploit(record, logger): + vulnerability = None + references = record.get("references", []) + for ref in references: + if ref.startswith("OSVDB") or ref.startswith("URL-"): + # ignore OSV-DB and reference exploit for metasploit + continue + + try: + if alias := Alias.objects.get(alias=ref): + vulnerability = alias.vulnerability + break + except Alias.DoesNotExist: + continue + + if not vulnerability: + logger(f"No vulnerability found for aliases {references}") + return 0 + + description = record.get("description", "") + notes = record.get("notes", {}) + platform = record.get("platform") + + source_url = "" + if path := record.get("path"): + source_url = f"https://github.com/rapid7/metasploit-framework/tree/master{path}" + source_date_published = None + + if disclosure_date := record.get("disclosure_date"): + try: + source_date_published = dateparser.parse(disclosure_date).date() + except ValueError: + logger(f"Error while parsing date {disclosure_date}", level=logging.ERROR) + + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="Metasploit", + defaults={ + "description": description, + "notes": saneyaml.dump(notes), + "source_date_published": source_date_published, + "platform": platform, + "source_url": source_url, + }, + ) + return 1 diff --git a/vulnerabilities/pipelines/exploitdb.py b/vulnerabilities/pipelines/exploitdb.py deleted file mode 100644 index 0c3bdc458..000000000 --- a/vulnerabilities/pipelines/exploitdb.py +++ /dev/null @@ -1,95 +0,0 @@ -import csv -import io -import logging - -import requests - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.pipelines import VulnerableCodePipeline - -logger = logging.getLogger(__name__) - - -class ExploitDBImproverPipeline(VulnerableCodePipeline): - """ - ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with - the specified alias, and create or update the ref and ref-type accordingly. - """ - - exploit_data = None - - license_expression = "GPL-2.0" - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploit, - ) - - def fetch_exploits(self): - exploit_db_url = ( - "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" - ) - response = requests.get(exploit_db_url) - self.exploit_data = io.StringIO(response.text) - - def add_exploit(self): - csvreader = csv.reader(self.exploit_data) - - header = next(csvreader) - for row in csvreader: - - aliases = row[11].split(";") - - for raw_alias in aliases: - - alias = Alias.objects.get_or_none(alias=raw_alias) - if not alias: - continue - - vul = alias.vulnerability - if not vul: - continue - - self.add_exploit_references(row[11], row[16], row[1], vul) - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="Exploit-DB", - defaults={ - "date_added": row[header.index("date_added")], - "description": row[header.index("description")], - "known_ransomware_campaign_use": row[header.index("verified")], - "source_date_published": row[header.index("date_published")], - "exploit_type": row[header.index("type")], - "platform": row[header.index("platform")], - "source_date_updated": row[header.index("date_updated")], - "source_url": row[header.index("source_url")], - }, - ) - - def add_exploit_references(self, ref_id, direct_url, path, vul): - url_map = { - "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", - "direct_url": direct_url, - } - - for key, url in url_map.items(): - if url: - ref, created = VulnerabilityReference.objects.update_or_create( - url=url, - defaults={ - "reference_id": ref_id, - "reference_type": VulnerabilityReference.EXPLOIT, - }, - ) - - if created: - VulnerabilityRelatedReference.objects.get_or_create( - vulnerability=vul, - reference=ref, - ) diff --git a/vulnerabilities/pipelines/metasploit.py b/vulnerabilities/pipelines/metasploit.py deleted file mode 100644 index be1829ede..000000000 --- a/vulnerabilities/pipelines/metasploit.py +++ /dev/null @@ -1,78 +0,0 @@ -import logging - -import requests -import saneyaml - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.pipelines import VulnerableCodePipeline - -module_logger = logging.getLogger(__name__) - - -class MetasploitImproverPipeline(VulnerableCodePipeline): - """ - Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities - by their associated aliases, and create or update the corresponding Exploit instances. - """ - - metasploit_data = {} - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploits, - ) - - def fetch_exploits(self): - url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" - response = requests.get(url) - if response.status_code != 200: - self.log(f"Failed to fetch the Metasploit Exploits: {url}") - return - self.metasploit_data = response.json() - - def add_exploits(self): - for _, record in self.metasploit_data.items(): - vul = None - for ref in record.get("references", []): - if ref.startswith("OSVDB") or ref.startswith("URL-"): - # ignore OSV-DB and reference exploit for metasploit - continue - - if not vul: - try: - alias = Alias.objects.get(alias=ref) - except Alias.DoesNotExist: - continue - - if not alias.vulnerability: - continue - - vul = alias.vulnerability - - if not vul: - continue - - description = record.get("description", "") - notes = record.get("notes", {}) - source_date_published = record.get("disclosure_date") - platform = record.get("platform") - - path = record.get("path") - source_url = ( - f"https://github.com/rapid7/metasploit-framework/tree/master{path}" if path else "" - ) - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="Metasploit", - defaults={ - "description": description, - "notes": saneyaml.dump(notes), - "source_date_published": source_date_published, - "platform": platform, - "source_url": source_url, - }, - ) diff --git a/vulnerabilities/pipelines/vulnerability_kev.py b/vulnerabilities/pipelines/vulnerability_kev.py deleted file mode 100644 index 255249472..000000000 --- a/vulnerabilities/pipelines/vulnerability_kev.py +++ /dev/null @@ -1,69 +0,0 @@ -import logging - -from sphinx.util import requests - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.pipelines import VulnerableCodePipeline - -module_logger = logging.getLogger(__name__) - - -class VulnerabilityKevPipeline(VulnerableCodePipeline): - """ - Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities - by their associated aliases, and create or update the corresponding Exploit instances. - """ - - kev_data = {} - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploits, - ) - - def fetch_exploits(self): - kev_url = ( - "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" - ) - response = requests.get(kev_url) - if response.status_code != 200: - self.log( - f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" - ) - return - self.kev_data = response.json() - - def add_exploits(self): - for kev_vul in self.kev_data.get("vulnerabilities", []): - cve_id = kev_vul.get("cveID") - - if not cve_id: - continue - - alias = Alias.objects.get_or_none(alias=cve_id) - - if not alias: - continue - - vul = alias.vulnerability - - if not vul: - continue - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="KEV", - defaults={ - "description": kev_vul["shortDescription"], - "date_added": kev_vul["dateAdded"], - "required_action": kev_vul["requiredAction"], - "due_date": kev_vul["dueDate"], - "notes": kev_vul["notes"], - "known_ransomware_campaign_use": True - if kev_vul["knownRansomwareCampaignUse"] == "Known" - else False, - }, - ) diff --git a/vulnerabilities/tests/pipelines/test_exploitdb.py b/vulnerabilities/tests/pipelines/test_exploitdb.py index 3f30433e5..f08f7fec0 100644 --- a/vulnerabilities/tests/pipelines/test_exploitdb.py +++ b/vulnerabilities/tests/pipelines/test_exploitdb.py @@ -7,7 +7,7 @@ from vulnerabilities.models import Alias from vulnerabilities.models import Exploit from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines.exploitdb import ExploitDBImproverPipeline +from vulnerabilities.pipelines.enhance_with_exploitdb import ExploitDBImproverPipeline BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "../test_data", "exploitdb_improver/files_exploits.csv") diff --git a/vulnerabilities/tests/pipelines/test_kev.py b/vulnerabilities/tests/pipelines/test_kev.py index e8931ff1a..71583a617 100644 --- a/vulnerabilities/tests/pipelines/test_kev.py +++ b/vulnerabilities/tests/pipelines/test_kev.py @@ -7,7 +7,7 @@ from vulnerabilities.models import Alias from vulnerabilities.models import Exploit from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines.vulnerability_kev import VulnerabilityKevPipeline +from vulnerabilities.pipelines.enhance_with_kev import VulnerabilityKevPipeline from vulnerabilities.utils import load_json BASE_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/vulnerabilities/tests/pipelines/test_metasploit.py b/vulnerabilities/tests/pipelines/test_metasploit.py index 65e893cba..1116950d2 100644 --- a/vulnerabilities/tests/pipelines/test_metasploit.py +++ b/vulnerabilities/tests/pipelines/test_metasploit.py @@ -7,7 +7,7 @@ from vulnerabilities.models import Alias from vulnerabilities.models import Exploit from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines.metasploit import MetasploitImproverPipeline +from vulnerabilities.pipelines.enhance_with_metasploit import MetasploitImproverPipeline from vulnerabilities.utils import load_json BASE_DIR = os.path.dirname(os.path.abspath(__file__)) From fbde42ffca958a088bcc65d165657d204d2b8970 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 17 Sep 2024 17:59:38 +0300 Subject: [PATCH 03/15] Resolve migration conflicts. Address the exploit in the API extension. Signed-off-by: ziadhany --- vulnerabilities/api_extension.py | 19 ++++++++++++++++--- vulnerabilities/improvers/__init__.py | 8 ++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/api_extension.py b/vulnerabilities/api_extension.py index a974f0796..4b9211c76 100644 --- a/vulnerabilities/api_extension.py +++ b/vulnerabilities/api_extension.py @@ -26,7 +26,7 @@ from rest_framework.throttling import AnonRateThrottle from vulnerabilities.api import BaseResourceSerializer -from vulnerabilities.models import Kev +from vulnerabilities.models import Exploit from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference @@ -105,8 +105,21 @@ class Meta: class V2ExploitSerializer(ModelSerializer): class Meta: - model = Kev - fields = ("description", "required_action", "date_added", "due_date", "resources_and_notes") + model = Exploit + fields = [ + "date_added", + "description", + "required_action", + "due_date", + "notes", + "known_ransomware_campaign_use", + "source_date_published", + "exploit_type", + "platform", + "source_date_updated", + "data_source", + "source_url", + ] class V2VulnerabilitySerializer(ModelSerializer): diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 20e437ab5..3e5febb60 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -8,8 +8,10 @@ # from vulnerabilities.improvers import valid_versions -from vulnerabilities.improvers import vulnerability_kev from vulnerabilities.improvers import vulnerability_status +from vulnerabilities.pipelines import enhance_with_exploitdb +from vulnerabilities.pipelines import enhance_with_kev +from vulnerabilities.pipelines import enhance_with_metasploit from vulnerabilities.pipelines import flag_ghost_packages IMPROVERS_REGISTRY = [ @@ -30,8 +32,10 @@ valid_versions.GithubOSVImprover, vulnerability_status.VulnerabilityStatusImprover, valid_versions.CurlImprover, - vulnerability_kev.VulnerabilityKevImprover, flag_ghost_packages.FlagGhostPackagePipeline, + enhance_with_kev.VulnerabilityKevPipeline, + enhance_with_metasploit.MetasploitImproverPipeline, + enhance_with_exploitdb.ExploitDBImproverPipeline, ] IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY} From 7165be6ec53b185d05143d75397cd9d8800c2484 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Sat, 21 Sep 2024 02:28:50 +0300 Subject: [PATCH 04/15] Add any missing logs message Remove unused logging module Signed-off-by: ziadhany --- vulnerabilities/pipelines/enhance_with_exploitdb.py | 2 ++ vulnerabilities/pipelines/enhance_with_kev.py | 2 -- vulnerabilities/pipelines/enhance_with_metasploit.py | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py index 70ccd31f1..41abece7a 100644 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -47,6 +47,7 @@ def fetch_exploits(self): self.exploit_data = io.StringIO(response.text) def add_exploit(self): + csvreader = csv.reader(self.exploit_data) header = next(csvreader) @@ -54,6 +55,7 @@ def add_exploit(self): fetched_exploit_count = len(raw_data) vulnerability_exploit_count = 0 + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) for row in progress.iter(raw_data): diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index 98732a721..b4d84eca7 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -7,8 +7,6 @@ from vulnerabilities.models import Exploit from vulnerabilities.pipelines import VulnerableCodePipeline -module_logger = logging.getLogger(__name__) - class VulnerabilityKevPipeline(VulnerableCodePipeline): """ diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index a569249d6..d3ee6e1a7 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -9,8 +9,6 @@ from vulnerabilities.models import Exploit from vulnerabilities.pipelines import VulnerableCodePipeline -module_logger = logging.getLogger(__name__) - class MetasploitImproverPipeline(VulnerableCodePipeline): """ From e0a0323119a2c1cf8e26e321079762ecbec6bbb9 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 20 Aug 2024 15:19:18 +0300 Subject: [PATCH 05/15] Migrate ( metasploit, exploit-db, kev ) to aboutcode pipeline. Set data_source as the header for the exploit table. Squash the migration files into a single file. Add test for exploit-db , metasploit Add a missing migration file Rename resources_and_notes to notes Fix Api test Refactor metasploit , exploitdb , kev improver Rename Kev tab to exploit tab Add support for exploitdb , metasploit, kev Signed-off-by: ziadhany --- vulnerabilities/api.py | 29 ++-- .../improvers/vulnerability_kev.py | 66 ------- .../migrations/0063_exploit_delete_kev.py | 131 ++++++++++++++ vulnerabilities/models.py | 73 ++++++-- vulnerabilities/pipelines/exploitdb.py | 95 ++++++++++ vulnerabilities/pipelines/metasploit.py | 78 +++++++++ .../pipelines/vulnerability_kev.py | 69 ++++++++ .../templates/vulnerability_details.html | 162 +++++++++++++----- .../tests/pipelines/test_exploitdb.py | 38 ++++ .../test_kev.py} | 27 +-- .../tests/pipelines/test_metasploit.py | 35 ++++ vulnerabilities/tests/test_api.py | 2 + .../exploitdb_improver/files_exploits.csv | 2 + .../modules_metadata_base.json | 93 ++++++++++ 14 files changed, 744 insertions(+), 156 deletions(-) create mode 100644 vulnerabilities/migrations/0063_exploit_delete_kev.py create mode 100644 vulnerabilities/pipelines/exploitdb.py create mode 100644 vulnerabilities/pipelines/metasploit.py create mode 100644 vulnerabilities/pipelines/vulnerability_kev.py create mode 100644 vulnerabilities/tests/pipelines/test_exploitdb.py rename vulnerabilities/tests/{test_kev_improver.py => pipelines/test_kev.py} (50%) create mode 100644 vulnerabilities/tests/pipelines/test_metasploit.py create mode 100644 vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv create mode 100644 vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 278ed636c..0b3dc2b8e 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -27,7 +27,7 @@ from rest_framework.throttling import UserRateThrottle from vulnerabilities.models import Alias -from vulnerabilities.models import Kev +from vulnerabilities.models import Exploit from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference @@ -175,10 +175,23 @@ def to_representation(self, instance): return representation -class KEVSerializer(serializers.ModelSerializer): +class ExploitSerializer(serializers.ModelSerializer): class Meta: - model = Kev - fields = ["date_added", "description", "required_action", "due_date", "resources_and_notes"] + model = Exploit + fields = [ + "date_added", + "description", + "required_action", + "due_date", + "notes", + "known_ransomware_campaign_use", + "source_date_published", + "exploit_type", + "platform", + "source_date_updated", + "data_source", + "source_url", + ] class VulnerabilitySerializer(BaseResourceSerializer): @@ -189,7 +202,7 @@ class VulnerabilitySerializer(BaseResourceSerializer): references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") aliases = AliasSerializer(many=True, source="alias") - kev = KEVSerializer(read_only=True) + exploits = ExploitSerializer(many=True, read_only=True) weaknesses = WeaknessSerializer(many=True) severity_range_score = serializers.SerializerMethodField() @@ -199,10 +212,6 @@ def to_representation(self, instance): weaknesses = data.get("weaknesses", []) data["weaknesses"] = [weakness for weakness in weaknesses if weakness is not None] - kev = data.get("kev", None) - if not kev: - data.pop("kev") - return data def get_severity_range_score(self, instance): @@ -240,7 +249,7 @@ class Meta: "affected_packages", "references", "weaknesses", - "kev", + "exploits", "severity_range_score", ] diff --git a/vulnerabilities/improvers/vulnerability_kev.py b/vulnerabilities/improvers/vulnerability_kev.py index 3ca3291bc..e69de29bb 100644 --- a/vulnerabilities/improvers/vulnerability_kev.py +++ b/vulnerabilities/improvers/vulnerability_kev.py @@ -1,66 +0,0 @@ -import logging -from typing import Iterable - -import requests -from django.db.models import QuerySet - -from vulnerabilities.improver import Improver -from vulnerabilities.improver import Inference -from vulnerabilities.models import Advisory -from vulnerabilities.models import Alias -from vulnerabilities.models import Kev - -logger = logging.getLogger(__name__) - - -class VulnerabilityKevImprover(Improver): - """ - Known Exploited Vulnerabilities Improver - """ - - @property - def interesting_advisories(self) -> QuerySet: - # TODO Modify KEV improver to iterate over the vulnerabilities alias, not the advisory - return [Advisory.objects.first()] - - def get_inferences(self, advisory_data) -> Iterable[Inference]: - """ - Fetch Kev data, iterate over it to find the vulnerability with the specified alias, and create or update - the Kev instance accordingly. - """ - - kev_url = ( - "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" - ) - response = requests.get(kev_url) - kev_data = response.json() - if response.status_code != 200: - logger.error( - f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" - ) - return [] - - for kev_vul in kev_data.get("vulnerabilities", []): - alias = Alias.objects.get_or_none(alias=kev_vul["cveID"]) - if not alias: - continue - - vul = alias.vulnerability - - if not vul: - continue - - Kev.objects.update_or_create( - vulnerability=vul, - defaults={ - "description": kev_vul["shortDescription"], - "date_added": kev_vul["dateAdded"], - "required_action": kev_vul["requiredAction"], - "due_date": kev_vul["dueDate"], - "resources_and_notes": kev_vul["notes"], - "known_ransomware_campaign_use": True - if kev_vul["knownRansomwareCampaignUse"] == "Known" - else False, - }, - ) - return [] diff --git a/vulnerabilities/migrations/0063_exploit_delete_kev.py b/vulnerabilities/migrations/0063_exploit_delete_kev.py new file mode 100644 index 000000000..00d2d60fe --- /dev/null +++ b/vulnerabilities/migrations/0063_exploit_delete_kev.py @@ -0,0 +1,131 @@ +# Generated by Django 4.1.13 on 2024-09-10 18:40 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0062_package_is_ghost"), + ] + + operations = [ + migrations.CreateModel( + name="Exploit", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "date_added", + models.DateField( + blank=True, + help_text="The date the vulnerability was added to an exploit catalog.", + null=True, + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", + null=True, + ), + ), + ( + "required_action", + models.TextField( + blank=True, + help_text="The required action to address the vulnerability, typically to apply vendor updates or apply vendor mitigations or to discontinue use.", + null=True, + ), + ), + ( + "due_date", + models.DateField( + blank=True, + help_text="The date the required action is due, which applies to all USA federal civilian executive branch (FCEB) agencies, but all organizations are strongly encouraged to execute the required action", + null=True, + ), + ), + ( + "notes", + models.TextField( + blank=True, + help_text="Additional notes and resources about the vulnerability, often a URL to vendor instructions.", + null=True, + ), + ), + ( + "known_ransomware_campaign_use", + models.BooleanField( + default=False, + help_text="Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; \n or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.", + ), + ), + ( + "source_date_published", + models.DateField( + blank=True, + help_text="The date that the exploit was published or disclosed.", + null=True, + ), + ), + ( + "exploit_type", + models.TextField( + blank=True, + help_text="The type of the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "platform", + models.TextField( + blank=True, + help_text="The platform associated with the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "source_date_updated", + models.DateField( + blank=True, + help_text="The date the exploit was updated in the original upstream data source.", + null=True, + ), + ), + ( + "data_source", + models.TextField( + blank=True, + help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", + null=True, + ), + ), + ( + "source_url", + models.URLField( + blank=True, + help_text="The URL to the exploit as provided in the original upstream data source.", + null=True, + ), + ), + ( + "vulnerability", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="exploits", + to="vulnerabilities.vulnerability", + ), + ), + ], + ), + migrations.DeleteModel( + name="Kev", + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index ada9bec54..7b46bbe50 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -1389,49 +1389,90 @@ def log_fixing(cls, package, importer, source_url, related_vulnerability): ) -class Kev(models.Model): +class Exploit(models.Model): """ - Known Exploited Vulnerabilities + A vulnerability exploit is code used to + take advantage of a security flaw for unauthorized access or malicious activity. """ - vulnerability = models.OneToOneField( + vulnerability = models.ForeignKey( Vulnerability, + related_name="exploits", on_delete=models.CASCADE, - related_name="kev", ) date_added = models.DateField( - help_text="The date the vulnerability was added to the Known Exploited Vulnerabilities" - " (KEV) catalog in the format YYYY-MM-DD.", null=True, blank=True, + help_text="The date the vulnerability was added to an exploit catalog.", ) description = models.TextField( - help_text="Description of the vulnerability in the Known Exploited Vulnerabilities" - " (KEV) catalog, usually a refinement of the original CVE description" + null=True, + blank=True, + help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", ) required_action = models.TextField( + null=True, + blank=True, help_text="The required action to address the vulnerability, typically to " - "apply vendor updates or apply vendor mitigations or to discontinue use." + "apply vendor updates or apply vendor mitigations or to discontinue use.", ) due_date = models.DateField( - help_text="The date the required action is due in the format YYYY-MM-DD," - "which applies to all USA federal civilian executive branch (FCEB) agencies," - "but all organizations are strongly encouraged to execute the required action." + null=True, + blank=True, + help_text="The date the required action is due, which applies" + " to all USA federal civilian executive branch (FCEB) agencies, " + "but all organizations are strongly encouraged to execute the required action", ) - resources_and_notes = models.TextField( + notes = models.TextField( + null=True, + blank=True, help_text="Additional notes and resources about the vulnerability," - " often a URL to vendor instructions." + " often a URL to vendor instructions.", ) known_ransomware_campaign_use = models.BooleanField( default=False, - help_text="""Known if this vulnerability is known to have been leveraged as part of a ransomware campaign; - or 'Unknown' if CISA lacks confirmation that the vulnerability has been utilized for ransomware.""", + help_text="""Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; + or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.""", + ) + + source_date_published = models.DateField( + null=True, blank=True, help_text="The date that the exploit was published or disclosed." + ) + + exploit_type = models.TextField( + null=True, + blank=True, + help_text="The type of the exploit as provided by the original upstream data source.", + ) + + platform = models.TextField( + null=True, + blank=True, + help_text="The platform associated with the exploit as provided by the original upstream data source.", + ) + + source_date_updated = models.DateField( + null=True, + blank=True, + help_text="The date the exploit was updated in the original upstream data source.", + ) + + data_source = models.TextField( + null=True, + blank=True, + help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", + ) + + source_url = models.URLField( + null=True, + blank=True, + help_text="The URL to the exploit as provided in the original upstream data source.", ) @property diff --git a/vulnerabilities/pipelines/exploitdb.py b/vulnerabilities/pipelines/exploitdb.py new file mode 100644 index 000000000..0c3bdc458 --- /dev/null +++ b/vulnerabilities/pipelines/exploitdb.py @@ -0,0 +1,95 @@ +import csv +import io +import logging + +import requests + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.pipelines import VulnerableCodePipeline + +logger = logging.getLogger(__name__) + + +class ExploitDBImproverPipeline(VulnerableCodePipeline): + """ + ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with + the specified alias, and create or update the ref and ref-type accordingly. + """ + + exploit_data = None + + license_expression = "GPL-2.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploit, + ) + + def fetch_exploits(self): + exploit_db_url = ( + "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" + ) + response = requests.get(exploit_db_url) + self.exploit_data = io.StringIO(response.text) + + def add_exploit(self): + csvreader = csv.reader(self.exploit_data) + + header = next(csvreader) + for row in csvreader: + + aliases = row[11].split(";") + + for raw_alias in aliases: + + alias = Alias.objects.get_or_none(alias=raw_alias) + if not alias: + continue + + vul = alias.vulnerability + if not vul: + continue + + self.add_exploit_references(row[11], row[16], row[1], vul) + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="Exploit-DB", + defaults={ + "date_added": row[header.index("date_added")], + "description": row[header.index("description")], + "known_ransomware_campaign_use": row[header.index("verified")], + "source_date_published": row[header.index("date_published")], + "exploit_type": row[header.index("type")], + "platform": row[header.index("platform")], + "source_date_updated": row[header.index("date_updated")], + "source_url": row[header.index("source_url")], + }, + ) + + def add_exploit_references(self, ref_id, direct_url, path, vul): + url_map = { + "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", + "direct_url": direct_url, + } + + for key, url in url_map.items(): + if url: + ref, created = VulnerabilityReference.objects.update_or_create( + url=url, + defaults={ + "reference_id": ref_id, + "reference_type": VulnerabilityReference.EXPLOIT, + }, + ) + + if created: + VulnerabilityRelatedReference.objects.get_or_create( + vulnerability=vul, + reference=ref, + ) diff --git a/vulnerabilities/pipelines/metasploit.py b/vulnerabilities/pipelines/metasploit.py new file mode 100644 index 000000000..be1829ede --- /dev/null +++ b/vulnerabilities/pipelines/metasploit.py @@ -0,0 +1,78 @@ +import logging + +import requests +import saneyaml + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class MetasploitImproverPipeline(VulnerableCodePipeline): + """ + Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + metasploit_data = {} + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" + response = requests.get(url) + if response.status_code != 200: + self.log(f"Failed to fetch the Metasploit Exploits: {url}") + return + self.metasploit_data = response.json() + + def add_exploits(self): + for _, record in self.metasploit_data.items(): + vul = None + for ref in record.get("references", []): + if ref.startswith("OSVDB") or ref.startswith("URL-"): + # ignore OSV-DB and reference exploit for metasploit + continue + + if not vul: + try: + alias = Alias.objects.get(alias=ref) + except Alias.DoesNotExist: + continue + + if not alias.vulnerability: + continue + + vul = alias.vulnerability + + if not vul: + continue + + description = record.get("description", "") + notes = record.get("notes", {}) + source_date_published = record.get("disclosure_date") + platform = record.get("platform") + + path = record.get("path") + source_url = ( + f"https://github.com/rapid7/metasploit-framework/tree/master{path}" if path else "" + ) + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="Metasploit", + defaults={ + "description": description, + "notes": saneyaml.dump(notes), + "source_date_published": source_date_published, + "platform": platform, + "source_url": source_url, + }, + ) diff --git a/vulnerabilities/pipelines/vulnerability_kev.py b/vulnerabilities/pipelines/vulnerability_kev.py new file mode 100644 index 000000000..255249472 --- /dev/null +++ b/vulnerabilities/pipelines/vulnerability_kev.py @@ -0,0 +1,69 @@ +import logging + +from sphinx.util import requests + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class VulnerabilityKevPipeline(VulnerableCodePipeline): + """ + Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + kev_data = {} + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + kev_url = ( + "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + ) + response = requests.get(kev_url) + if response.status_code != 200: + self.log( + f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" + ) + return + self.kev_data = response.json() + + def add_exploits(self): + for kev_vul in self.kev_data.get("vulnerabilities", []): + cve_id = kev_vul.get("cveID") + + if not cve_id: + continue + + alias = Alias.objects.get_or_none(alias=cve_id) + + if not alias: + continue + + vul = alias.vulnerability + + if not vul: + continue + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="KEV", + defaults={ + "description": kev_vul["shortDescription"], + "date_added": kev_vul["dateAdded"], + "required_action": kev_vul["requiredAction"], + "due_date": kev_vul["dueDate"], + "notes": kev_vul["notes"], + "known_ransomware_campaign_use": True + if kev_vul["knownRansomwareCampaignUse"] == "Known" + else False, + }, + ) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index c950adad1..aeb3eb649 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -61,11 +61,11 @@
    • - {% if vulnerability.kev %} -
  • + {% if vulnerability.exploits %} +
  • - Known Exploited Vulnerabilities + Exploits ({{ vulnerability.exploits.count }})
    • @@ -77,7 +77,7 @@ EPSS - +
  • @@ -439,87 +439,157 @@ {% endfor %} - {% if vulnerability.kev %} -
    -
    - Known Exploited Vulnerabilities -
    - + + +
    + {% for exploit in vulnerability.exploits.all %} +
    + + + + + - - - - - - {% if vulnerability.kev.description %} + {% if exploit.date_added %} + + + + + {% endif %} + {% if exploit.description %} - + + + {% endif %} + {% if exploit.required_action %} + + + + + {% endif %} + {% if exploit.due_date %} + + + + + {% endif %} + {% if exploit.notes %} + + + + + {% endif %} + {% if exploit.known_ransomware_campaign_use %} + + + {% endif %} - {% if vulnerability.kev.required_action %} + {% if exploit.source_date_published %} - + {% endif %} - - {% if vulnerability.kev.resources_and_notes %} + {% if exploit.exploit_type %} - + {% endif %} - - {% if vulnerability.kev.due_date %} + {% if exploit.platform %} - + {% endif %} - {% if vulnerability.kev.date_added %} + {% if exploit.source_date_updated %} - + + + {% endif %} + + {% if exploit.source_url %} + + + {% endif %} - -
    data_source {{ exploit.data_source }}
    - - Known Ransomware Campaign Use: - - {{ vulnerability.kev.get_known_ransomware_campaign_use_type }}
    + + date_added: + + {{ exploit.date_added }}
    + data-tooltip="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description"> Description: {{ vulnerability.kev.description }}{{ exploit.description }}
    + + required_action: + + {{ exploit.required_action }}
    + + due_date: + + {{ exploit.due_date }}
    + + notes: + + 
    {{ exploit.notes }}
    + + known_ransomware_campaign_use: + + {{ exploit.known_ransomware_campaign_use }}
    - Required Action: + data-tooltip="The date that the exploit was published or disclosed."> + source_date_published: {{ vulnerability.kev.required_action }}{{ exploit.source_date_published }}
    - Notes: + data-tooltip="The type of the exploit as provided by the original upstream data source."> + exploit_type: {{ vulnerability.kev.resources_and_notes }}{{ exploit.exploit_type }}
    - Due Date: + data-tooltip="The platform associated with the exploit as provided by the original upstream data source."> + platform: {{ vulnerability.kev.due_date }}{{ exploit.platform }}
    - Date Added: + data-tooltip="The date the exploit was updated in the original upstream data source."> + source_date_updated: {{ vulnerability.kev.date_added }}{{ exploit.source_date_updated }}
    + + source_url: + + {{ exploit.source_url }}
    -
    - {% endif %} + + {% empty %} + + + No exploits are available. + + + {% endfor %} + + {% for severity in severities %} {% if severity.scoring_system == 'epss' %} diff --git a/vulnerabilities/tests/pipelines/test_exploitdb.py b/vulnerabilities/tests/pipelines/test_exploitdb.py new file mode 100644 index 000000000..3f30433e5 --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_exploitdb.py @@ -0,0 +1,38 @@ +import os +from unittest import mock +from unittest.mock import Mock + +import pytest + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.exploitdb import ExploitDBImproverPipeline + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "exploitdb_improver/files_exploits.csv") + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_exploit_db_improver(mock_get): + mock_response = Mock(status_code=200) + with open(TEST_DATA, "r") as f: + mock_response.text = f.read() + mock_get.return_value = mock_response + + improver = ExploitDBImproverPipeline() + + # Run the improver when there is no matching aliases + improver.execute() + + assert Exploit.objects.count() == 0 + + v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") + v1.save() + + Alias.objects.create(alias="CVE-2009-3699", vulnerability=v1) + + # Run Exploit-DB Improver again when there are matching aliases. + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/test_kev_improver.py b/vulnerabilities/tests/pipelines/test_kev.py similarity index 50% rename from vulnerabilities/tests/test_kev_improver.py rename to vulnerabilities/tests/pipelines/test_kev.py index d0b1c981a..e8931ff1a 100644 --- a/vulnerabilities/tests/test_kev_improver.py +++ b/vulnerabilities/tests/pipelines/test_kev.py @@ -1,41 +1,32 @@ import os -from datetime import datetime from unittest import mock from unittest.mock import Mock import pytest -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.improvers.vulnerability_kev import VulnerabilityKevImprover from vulnerabilities.models import Alias -from vulnerabilities.models import Kev +from vulnerabilities.models import Exploit from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.vulnerability_kev import VulnerabilityKevPipeline from vulnerabilities.utils import load_json BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "kev_data.json") +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "kev_data.json") @pytest.mark.django_db @mock.patch("requests.get") def test_kev_improver(mock_get): - advisory_data = AdvisoryData( - aliases=["CVE-2022-21831"], - summary="Possible code injection vulnerability in Rails / Active Storage", - affected_packages=[], - references=[], - date_published=datetime.now(), - ) # to just run the improver - mock_response = Mock(status_code=200) mock_response.json.return_value = load_json(TEST_DATA) mock_get.return_value = mock_response - improver = VulnerabilityKevImprover() + improver = VulnerabilityKevPipeline() # Run the improver when there is no matching aliases - improver.get_inferences(advisory_data=advisory_data) - assert Kev.objects.count() == 0 + improver.execute() + + assert Exploit.objects.count() == 0 v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") v1.save() @@ -43,5 +34,5 @@ def test_kev_improver(mock_get): Alias.objects.create(alias="CVE-2021-38647", vulnerability=v1) # Run Kev Improver again when there are matching aliases. - improver.get_inferences(advisory_data=advisory_data) - assert Kev.objects.count() == 1 + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/pipelines/test_metasploit.py b/vulnerabilities/tests/pipelines/test_metasploit.py new file mode 100644 index 000000000..65e893cba --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_metasploit.py @@ -0,0 +1,35 @@ +import os +from unittest import mock +from unittest.mock import Mock + +import pytest + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.metasploit import MetasploitImproverPipeline +from vulnerabilities.utils import load_json + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "metasploit_improver/modules_metadata_base.json") + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_metasploit_improver(mock_get): + mock_response = Mock(status_code=200) + mock_response.json.return_value = load_json(TEST_DATA) + mock_get.return_value = mock_response + + improver = MetasploitImproverPipeline() + + # Run the improver when there is no matching aliases + improver.execute() + assert Exploit.objects.count() == 0 + + v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") + Alias.objects.create(alias="CVE-2007-4387", vulnerability=v1) + + # Run metasploit Improver again when there are matching aliases. + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 8fb50243a..18807a6d7 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -296,6 +296,7 @@ def test_api_with_single_vulnerability(self): "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", }, ], + "exploits": [], } def test_api_with_single_vulnerability_with_filters(self): @@ -341,6 +342,7 @@ def test_api_with_single_vulnerability_with_filters(self): "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", }, ], + "exploits": [], } diff --git a/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv b/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv new file mode 100644 index 000000000..a63701d8c --- /dev/null +++ b/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv @@ -0,0 +1,2 @@ +id,file,description,date_published,author,type,platform,port,date_added,date_updated,verified,codes,tags,aliases,screenshot_url,application_url,source_url +16929,exploits/aix/dos/16929.rb,"AIX Calendar Manager Service Daemon (rpc.cmsd) Opcode 21 - Buffer Overflow (Metasploit)",2010-11-11,Metasploit,dos,aix,,2010-11-11,2011-03-06,1,CVE-2009-3699;OSVDB-58726,"Metasploit Framework (MSF)",,,,http://aix.software.ibm.com/aix/efixes/security/cmsd_advisory.asc diff --git a/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json b/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json new file mode 100644 index 000000000..e9351a1df --- /dev/null +++ b/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json @@ -0,0 +1,93 @@ +{ + "auxiliary_admin/2wire/xslt_password_reset": { + "name": "2Wire Cross-Site Request Forgery Password Reset Vulnerability", + "fullname": "auxiliary/admin/2wire/xslt_password_reset", + "aliases": [ + ], + "rank": 300, + "disclosure_date": "2007-08-15", + "type": "auxiliary", + "author": [ + "hkm ", + "Travis Phillips" + ], + "description": "This module will reset the admin password on a 2Wire wireless router. This is\n done by using the /xslt page where authentication is not required, thus allowing\n configuration changes (such as resetting the password) as administrators.", + "references": [ + "CVE-2007-4387", + "OSVDB-37667", + "BID-36075", + "URL-https://seclists.org/bugtraq/2007/Aug/225" + ], + "platform": "", + "arch": "", + "rport": 80, + "autofilter_ports": [ + 80, + 8080, + 443, + 8000, + 8888, + 8880, + 8008, + 3000, + 8443 + ], + "autofilter_services": [ + "http", + "https" + ], + "targets": null, + "mod_time": "2020-10-02 17:38:06 +0000", + "path": "/modules/auxiliary/admin/2wire/xslt_password_reset.rb", + "is_install_path": true, + "ref_name": "admin/2wire/xslt_password_reset", + "check": false, + "post_auth": false, + "default_credential": false, + "notes": { + }, + "session_types": false, + "needs_cleanup": false, + "actions": [ + ] + }, + "post_firefox/manage/webcam_chat": { + "name": "Firefox Webcam Chat on Privileged Javascript Shell", + "fullname": "post/firefox/manage/webcam_chat", + "aliases": [ + + ], + "rank": 300, + "disclosure_date": "2014-05-13", + "type": "post", + "author": [ + "joev " + ], + "description": "This module allows streaming a webcam from a privileged Firefox Javascript shell.", + "references": [ + "URL-http://www.rapid7.com/db/modules/exploit/firefox/local/exec_shellcode" + ], + "platform": "", + "arch": "", + "rport": null, + "autofilter_ports": null, + "autofilter_services": null, + "targets": null, + "mod_time": "2023-02-08 13:47:34 +0000", + "path": "/modules/post/firefox/manage/webcam_chat.rb", + "is_install_path": true, + "ref_name": "firefox/manage/webcam_chat", + "check": false, + "post_auth": false, + "default_credential": false, + "notes": { + }, + "session_types": [ + + ], + "needs_cleanup": null, + "actions": [ + + ] + } +} \ No newline at end of file From c83facdd49ccededa476fa6b666097b15d046dbd Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 17 Sep 2024 17:48:51 +0300 Subject: [PATCH 06/15] Implement the appropriate LoopProgress progress bar. Refactor the error handling logic in the code. Signed-off-by: ziadhany --- .../pipelines/enhance_with_exploitdb.py | 149 ++++++++++++++++++ vulnerabilities/pipelines/enhance_with_kev.py | 82 ++++++++++ .../pipelines/enhance_with_metasploit.py | 103 ++++++++++++ vulnerabilities/pipelines/exploitdb.py | 95 ----------- vulnerabilities/pipelines/metasploit.py | 78 --------- .../pipelines/vulnerability_kev.py | 69 -------- .../tests/pipelines/test_exploitdb.py | 2 +- vulnerabilities/tests/pipelines/test_kev.py | 2 +- .../tests/pipelines/test_metasploit.py | 2 +- 9 files changed, 337 insertions(+), 245 deletions(-) create mode 100644 vulnerabilities/pipelines/enhance_with_exploitdb.py create mode 100644 vulnerabilities/pipelines/enhance_with_kev.py create mode 100644 vulnerabilities/pipelines/enhance_with_metasploit.py delete mode 100644 vulnerabilities/pipelines/exploitdb.py delete mode 100644 vulnerabilities/pipelines/metasploit.py delete mode 100644 vulnerabilities/pipelines/vulnerability_kev.py diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py new file mode 100644 index 000000000..70ccd31f1 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -0,0 +1,149 @@ +import csv +import io +import logging + +import requests +from aboutcode.pipeline import LoopProgress +from dateutil import parser as dateparser +from django.db import DataError + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class ExploitDBImproverPipeline(VulnerableCodePipeline): + """ + ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with + the specified alias, and create or update the ref and ref-type accordingly. + """ + + license_expression = "GPL-2.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploit, + ) + + def fetch_exploits(self): + exploit_db_url = ( + "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" + ) + + try: + response = requests.get(exploit_db_url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the Exploit-DB Exploits: {exploit_db_url} - {http_err}", + level=logging.ERROR, + ) + raise + + self.exploit_data = io.StringIO(response.text) + + def add_exploit(self): + csvreader = csv.reader(self.exploit_data) + header = next(csvreader) + + raw_data = list(csvreader) + fetched_exploit_count = len(raw_data) + + vulnerability_exploit_count = 0 + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + + for row in progress.iter(raw_data): + vulnerability_exploit_count += add_vulnerability_exploit(row, header, self.log) + + self.log( + f"Successfully added {vulnerability_exploit_count:,d} exploit-db vulnerability exploit" + ) + + +def add_vulnerability_exploit(row, header, logger): + vulnerability = None + aliases = row[11].split(";") + + for raw_alias in aliases: + try: + if alias := Alias.objects.get(alias=raw_alias): + vulnerability = alias.vulnerability + break + except Alias.DoesNotExist: + continue + + if not vulnerability: + logger(f"No vulnerability found for aliases {aliases}") + return 0 + + add_exploit_references(row[11], row[16], row[1], vulnerability, logger) + + date_added = parse_date(row[header.index("date_added")]) + source_date_published = parse_date(row[header.index("date_published")]) + source_date_updated = parse_date(row[header.index("date_updated")]) + + try: + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="Exploit-DB", + defaults={ + "date_added": date_added, + "description": row[header.index("description")], + "known_ransomware_campaign_use": row[header.index("verified")], + "source_date_published": source_date_published, + "exploit_type": row[header.index("type")], + "platform": row[header.index("platform")], + "source_date_updated": source_date_updated, + "source_url": row[header.index("source_url")], + }, + ) + except DataError as e: + logger( + f"Failed to Create the Vulnerability Exploit-DB: {e}", + level=logging.ERROR, + ) + return 1 + + +def add_exploit_references(ref_id, direct_url, path, vul, logger): + url_map = { + "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", + "direct_url": direct_url, + } + + for key, url in url_map.items(): + if url: + try: + ref, created = VulnerabilityReference.objects.update_or_create( + url=url, + defaults={ + "reference_id": ref_id, + "reference_type": VulnerabilityReference.EXPLOIT, + }, + ) + + if created: + VulnerabilityRelatedReference.objects.get_or_create( + vulnerability=vul, + reference=ref, + ) + + except DataError as e: + logger( + f"Failed to Create the Vulnerability Reference For Exploit-DB: {e}", + level=logging.ERROR, + ) + + +def parse_date(date_string): + if date_string: + try: + date_obj = dateparser.parse(date_string).date() + return date_obj.strftime("%Y-%m-%d") + except (ValueError, TypeError, Exception) as e: + logging.error(f"Error while parsing ExploitDB date '{date_string}': {e}") + return diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py new file mode 100644 index 000000000..98732a721 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -0,0 +1,82 @@ +import logging + +import requests +from aboutcode.pipeline import LoopProgress + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class VulnerabilityKevPipeline(VulnerableCodePipeline): + """ + Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + kev_url = ( + "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + ) + try: + response = requests.get(kev_url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the KEV Exploits: {kev_url} - {http_err}", + level=logging.ERROR, + ) + raise + self.kev_data = response.json() + + def add_exploits(self): + fetched_exploit_count = self.kev_data.get("count") + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") + + vulnerability_exploit_count = 0 + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + + for record in progress.iter(self.kev_data.get("vulnerabilities", [])): + vulnerability_exploit_count += add_vulnerability_exploit( + kev_vul=record, + logger=self.log, + ) + + self.log(f"Successfully added {vulnerability_exploit_count:,d} kev exploit") + + +def add_vulnerability_exploit(kev_vul, logger): + cve_id = kev_vul.get("cveID") + + vulnerability = None + try: + if alias := Alias.objects.get(alias=cve_id): + vulnerability = alias.vulnerability + except Alias.DoesNotExist: + logger(f"No vulnerability found for aliases {cve_id}") + return 0 + + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="KEV", + defaults={ + "description": kev_vul["shortDescription"], + "date_added": kev_vul["dateAdded"], + "required_action": kev_vul["requiredAction"], + "due_date": kev_vul["dueDate"], + "notes": kev_vul["notes"], + "known_ransomware_campaign_use": True + if kev_vul["knownRansomwareCampaignUse"] == "Known" + else False, + }, + ) + return 1 diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py new file mode 100644 index 000000000..a569249d6 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -0,0 +1,103 @@ +import logging + +import requests +import saneyaml +from aboutcode.pipeline import LoopProgress +from dateutil import parser as dateparser + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class MetasploitImproverPipeline(VulnerableCodePipeline): + """ + Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + license_expression = "BSD-3-clause" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_vulnerability_exploits, + ) + + def fetch_exploits(self): + url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" + self.log(f"Fetching {url}") + try: + response = requests.get(url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the Metasploit Exploits: {url} - {http_err}", level=logging.ERROR + ) + raise + + self.metasploit_data = response.json() + + def add_vulnerability_exploits(self): + fetched_exploit_count = len(self.metasploit_data) + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") + + vulnerability_exploit_count = 0 + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + for _, record in progress.iter(self.metasploit_data.items()): + vulnerability_exploit_count += add_vulnerability_exploit( + record=record, + logger=self.log, + ) + self.log(f"Successfully added {vulnerability_exploit_count:,d} vulnerability exploit") + + +def add_vulnerability_exploit(record, logger): + vulnerability = None + references = record.get("references", []) + for ref in references: + if ref.startswith("OSVDB") or ref.startswith("URL-"): + # ignore OSV-DB and reference exploit for metasploit + continue + + try: + if alias := Alias.objects.get(alias=ref): + vulnerability = alias.vulnerability + break + except Alias.DoesNotExist: + continue + + if not vulnerability: + logger(f"No vulnerability found for aliases {references}") + return 0 + + description = record.get("description", "") + notes = record.get("notes", {}) + platform = record.get("platform") + + source_url = "" + if path := record.get("path"): + source_url = f"https://github.com/rapid7/metasploit-framework/tree/master{path}" + source_date_published = None + + if disclosure_date := record.get("disclosure_date"): + try: + source_date_published = dateparser.parse(disclosure_date).date() + except ValueError: + logger(f"Error while parsing date {disclosure_date}", level=logging.ERROR) + + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="Metasploit", + defaults={ + "description": description, + "notes": saneyaml.dump(notes), + "source_date_published": source_date_published, + "platform": platform, + "source_url": source_url, + }, + ) + return 1 diff --git a/vulnerabilities/pipelines/exploitdb.py b/vulnerabilities/pipelines/exploitdb.py deleted file mode 100644 index 0c3bdc458..000000000 --- a/vulnerabilities/pipelines/exploitdb.py +++ /dev/null @@ -1,95 +0,0 @@ -import csv -import io -import logging - -import requests - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.pipelines import VulnerableCodePipeline - -logger = logging.getLogger(__name__) - - -class ExploitDBImproverPipeline(VulnerableCodePipeline): - """ - ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with - the specified alias, and create or update the ref and ref-type accordingly. - """ - - exploit_data = None - - license_expression = "GPL-2.0" - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploit, - ) - - def fetch_exploits(self): - exploit_db_url = ( - "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" - ) - response = requests.get(exploit_db_url) - self.exploit_data = io.StringIO(response.text) - - def add_exploit(self): - csvreader = csv.reader(self.exploit_data) - - header = next(csvreader) - for row in csvreader: - - aliases = row[11].split(";") - - for raw_alias in aliases: - - alias = Alias.objects.get_or_none(alias=raw_alias) - if not alias: - continue - - vul = alias.vulnerability - if not vul: - continue - - self.add_exploit_references(row[11], row[16], row[1], vul) - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="Exploit-DB", - defaults={ - "date_added": row[header.index("date_added")], - "description": row[header.index("description")], - "known_ransomware_campaign_use": row[header.index("verified")], - "source_date_published": row[header.index("date_published")], - "exploit_type": row[header.index("type")], - "platform": row[header.index("platform")], - "source_date_updated": row[header.index("date_updated")], - "source_url": row[header.index("source_url")], - }, - ) - - def add_exploit_references(self, ref_id, direct_url, path, vul): - url_map = { - "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", - "direct_url": direct_url, - } - - for key, url in url_map.items(): - if url: - ref, created = VulnerabilityReference.objects.update_or_create( - url=url, - defaults={ - "reference_id": ref_id, - "reference_type": VulnerabilityReference.EXPLOIT, - }, - ) - - if created: - VulnerabilityRelatedReference.objects.get_or_create( - vulnerability=vul, - reference=ref, - ) diff --git a/vulnerabilities/pipelines/metasploit.py b/vulnerabilities/pipelines/metasploit.py deleted file mode 100644 index be1829ede..000000000 --- a/vulnerabilities/pipelines/metasploit.py +++ /dev/null @@ -1,78 +0,0 @@ -import logging - -import requests -import saneyaml - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.pipelines import VulnerableCodePipeline - -module_logger = logging.getLogger(__name__) - - -class MetasploitImproverPipeline(VulnerableCodePipeline): - """ - Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities - by their associated aliases, and create or update the corresponding Exploit instances. - """ - - metasploit_data = {} - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploits, - ) - - def fetch_exploits(self): - url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" - response = requests.get(url) - if response.status_code != 200: - self.log(f"Failed to fetch the Metasploit Exploits: {url}") - return - self.metasploit_data = response.json() - - def add_exploits(self): - for _, record in self.metasploit_data.items(): - vul = None - for ref in record.get("references", []): - if ref.startswith("OSVDB") or ref.startswith("URL-"): - # ignore OSV-DB and reference exploit for metasploit - continue - - if not vul: - try: - alias = Alias.objects.get(alias=ref) - except Alias.DoesNotExist: - continue - - if not alias.vulnerability: - continue - - vul = alias.vulnerability - - if not vul: - continue - - description = record.get("description", "") - notes = record.get("notes", {}) - source_date_published = record.get("disclosure_date") - platform = record.get("platform") - - path = record.get("path") - source_url = ( - f"https://github.com/rapid7/metasploit-framework/tree/master{path}" if path else "" - ) - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="Metasploit", - defaults={ - "description": description, - "notes": saneyaml.dump(notes), - "source_date_published": source_date_published, - "platform": platform, - "source_url": source_url, - }, - ) diff --git a/vulnerabilities/pipelines/vulnerability_kev.py b/vulnerabilities/pipelines/vulnerability_kev.py deleted file mode 100644 index 255249472..000000000 --- a/vulnerabilities/pipelines/vulnerability_kev.py +++ /dev/null @@ -1,69 +0,0 @@ -import logging - -from sphinx.util import requests - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.pipelines import VulnerableCodePipeline - -module_logger = logging.getLogger(__name__) - - -class VulnerabilityKevPipeline(VulnerableCodePipeline): - """ - Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities - by their associated aliases, and create or update the corresponding Exploit instances. - """ - - kev_data = {} - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploits, - ) - - def fetch_exploits(self): - kev_url = ( - "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" - ) - response = requests.get(kev_url) - if response.status_code != 200: - self.log( - f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" - ) - return - self.kev_data = response.json() - - def add_exploits(self): - for kev_vul in self.kev_data.get("vulnerabilities", []): - cve_id = kev_vul.get("cveID") - - if not cve_id: - continue - - alias = Alias.objects.get_or_none(alias=cve_id) - - if not alias: - continue - - vul = alias.vulnerability - - if not vul: - continue - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="KEV", - defaults={ - "description": kev_vul["shortDescription"], - "date_added": kev_vul["dateAdded"], - "required_action": kev_vul["requiredAction"], - "due_date": kev_vul["dueDate"], - "notes": kev_vul["notes"], - "known_ransomware_campaign_use": True - if kev_vul["knownRansomwareCampaignUse"] == "Known" - else False, - }, - ) diff --git a/vulnerabilities/tests/pipelines/test_exploitdb.py b/vulnerabilities/tests/pipelines/test_exploitdb.py index 3f30433e5..f08f7fec0 100644 --- a/vulnerabilities/tests/pipelines/test_exploitdb.py +++ b/vulnerabilities/tests/pipelines/test_exploitdb.py @@ -7,7 +7,7 @@ from vulnerabilities.models import Alias from vulnerabilities.models import Exploit from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines.exploitdb import ExploitDBImproverPipeline +from vulnerabilities.pipelines.enhance_with_exploitdb import ExploitDBImproverPipeline BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "../test_data", "exploitdb_improver/files_exploits.csv") diff --git a/vulnerabilities/tests/pipelines/test_kev.py b/vulnerabilities/tests/pipelines/test_kev.py index e8931ff1a..71583a617 100644 --- a/vulnerabilities/tests/pipelines/test_kev.py +++ b/vulnerabilities/tests/pipelines/test_kev.py @@ -7,7 +7,7 @@ from vulnerabilities.models import Alias from vulnerabilities.models import Exploit from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines.vulnerability_kev import VulnerabilityKevPipeline +from vulnerabilities.pipelines.enhance_with_kev import VulnerabilityKevPipeline from vulnerabilities.utils import load_json BASE_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/vulnerabilities/tests/pipelines/test_metasploit.py b/vulnerabilities/tests/pipelines/test_metasploit.py index 65e893cba..1116950d2 100644 --- a/vulnerabilities/tests/pipelines/test_metasploit.py +++ b/vulnerabilities/tests/pipelines/test_metasploit.py @@ -7,7 +7,7 @@ from vulnerabilities.models import Alias from vulnerabilities.models import Exploit from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines.metasploit import MetasploitImproverPipeline +from vulnerabilities.pipelines.enhance_with_metasploit import MetasploitImproverPipeline from vulnerabilities.utils import load_json BASE_DIR = os.path.dirname(os.path.abspath(__file__)) From fe55cf27e3e305d23e500c7201e2b6dbb84a239c Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 17 Sep 2024 17:59:38 +0300 Subject: [PATCH 07/15] Resolve migration conflicts. Address the exploit in the API extension. Signed-off-by: ziadhany --- vulnerabilities/api_extension.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/api_extension.py b/vulnerabilities/api_extension.py index a974f0796..4b9211c76 100644 --- a/vulnerabilities/api_extension.py +++ b/vulnerabilities/api_extension.py @@ -26,7 +26,7 @@ from rest_framework.throttling import AnonRateThrottle from vulnerabilities.api import BaseResourceSerializer -from vulnerabilities.models import Kev +from vulnerabilities.models import Exploit from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference @@ -105,8 +105,21 @@ class Meta: class V2ExploitSerializer(ModelSerializer): class Meta: - model = Kev - fields = ("description", "required_action", "date_added", "due_date", "resources_and_notes") + model = Exploit + fields = [ + "date_added", + "description", + "required_action", + "due_date", + "notes", + "known_ransomware_campaign_use", + "source_date_published", + "exploit_type", + "platform", + "source_date_updated", + "data_source", + "source_url", + ] class V2VulnerabilitySerializer(ModelSerializer): From b7a31cdf70a3bf4451b66105ef8b0583dbbcd2d4 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Sat, 21 Sep 2024 02:28:50 +0300 Subject: [PATCH 08/15] Add any missing logs message Remove unused logging module Signed-off-by: ziadhany --- vulnerabilities/pipelines/enhance_with_exploitdb.py | 2 ++ vulnerabilities/pipelines/enhance_with_kev.py | 2 -- vulnerabilities/pipelines/enhance_with_metasploit.py | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py index 70ccd31f1..41abece7a 100644 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -47,6 +47,7 @@ def fetch_exploits(self): self.exploit_data = io.StringIO(response.text) def add_exploit(self): + csvreader = csv.reader(self.exploit_data) header = next(csvreader) @@ -54,6 +55,7 @@ def add_exploit(self): fetched_exploit_count = len(raw_data) vulnerability_exploit_count = 0 + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) for row in progress.iter(raw_data): diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index 98732a721..b4d84eca7 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -7,8 +7,6 @@ from vulnerabilities.models import Exploit from vulnerabilities.pipelines import VulnerableCodePipeline -module_logger = logging.getLogger(__name__) - class VulnerabilityKevPipeline(VulnerableCodePipeline): """ diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index a569249d6..d3ee6e1a7 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -9,8 +9,6 @@ from vulnerabilities.models import Exploit from vulnerabilities.pipelines import VulnerableCodePipeline -module_logger = logging.getLogger(__name__) - class MetasploitImproverPipeline(VulnerableCodePipeline): """ From 14fc92097721479e992f4efdd3ec45bcc947182f Mon Sep 17 00:00:00 2001 From: ziadhany Date: Sat, 21 Sep 2024 19:00:21 +0300 Subject: [PATCH 09/15] Fix migration conflict Add pipeline_id for ( kev, metasploit, exploit-db ) Signed-off-by: ziadhany --- vulnerabilities/improvers/__init__.py | 8 ++++++-- ...3_exploit_delete_kev.py => 0065_exploit_delete_kev.py} | 4 ++-- vulnerabilities/pipelines/enhance_with_exploitdb.py | 3 ++- vulnerabilities/pipelines/enhance_with_kev.py | 2 ++ vulnerabilities/pipelines/enhance_with_metasploit.py | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-) rename vulnerabilities/migrations/{0063_exploit_delete_kev.py => 0065_exploit_delete_kev.py} (97%) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index d15504166..6e9c24b38 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -8,9 +8,11 @@ # from vulnerabilities.improvers import valid_versions -from vulnerabilities.improvers import vulnerability_kev from vulnerabilities.improvers import vulnerability_status from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.pipelines import enhance_with_exploitdb +from vulnerabilities.pipelines import enhance_with_kev +from vulnerabilities.pipelines import enhance_with_metasploit from vulnerabilities.pipelines import flag_ghost_packages IMPROVERS_REGISTRY = [ @@ -31,8 +33,10 @@ valid_versions.GithubOSVImprover, vulnerability_status.VulnerabilityStatusImprover, valid_versions.CurlImprover, - vulnerability_kev.VulnerabilityKevImprover, flag_ghost_packages.FlagGhostPackagePipeline, + enhance_with_kev.VulnerabilityKevPipeline, + enhance_with_metasploit.MetasploitImproverPipeline, + enhance_with_exploitdb.ExploitDBImproverPipeline, ] IMPROVERS_REGISTRY = { diff --git a/vulnerabilities/migrations/0063_exploit_delete_kev.py b/vulnerabilities/migrations/0065_exploit_delete_kev.py similarity index 97% rename from vulnerabilities/migrations/0063_exploit_delete_kev.py rename to vulnerabilities/migrations/0065_exploit_delete_kev.py index 00d2d60fe..28b14d2c9 100644 --- a/vulnerabilities/migrations/0063_exploit_delete_kev.py +++ b/vulnerabilities/migrations/0065_exploit_delete_kev.py @@ -1,4 +1,4 @@ -# Generated by Django 4.1.13 on 2024-09-10 18:40 +# Generated by Django 4.2.15 on 2024-09-21 15:37 from django.db import migrations, models import django.db.models.deletion @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ("vulnerabilities", "0062_package_is_ghost"), + ("vulnerabilities", "0064_update_npm_pypa_advisory_created_by"), ] operations = [ diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py index 41abece7a..eadf7919f 100644 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -20,7 +20,8 @@ class ExploitDBImproverPipeline(VulnerableCodePipeline): the specified alias, and create or update the ref and ref-type accordingly. """ - license_expression = "GPL-2.0" + pipeline_id = "enhance_with_exploitdb" + spdx_license_expression = "GPL-2.0" @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index b4d84eca7..8ec801c5c 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -14,6 +14,8 @@ class VulnerabilityKevPipeline(VulnerableCodePipeline): by their associated aliases, and create or update the corresponding Exploit instances. """ + pipeline_id = "enhance_with_kev" + @classmethod def steps(cls): return ( diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index d3ee6e1a7..8bb2c79cc 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -16,7 +16,8 @@ class MetasploitImproverPipeline(VulnerableCodePipeline): by their associated aliases, and create or update the corresponding Exploit instances. """ - license_expression = "BSD-3-clause" + pipeline_id = "enhance_with_metasploit" + spdx_license_expression = "BSD-3-clause" @classmethod def steps(cls): From 8136e9a15a6a176d52457ccab7a8d4c1b2c3f3d1 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Sat, 21 Sep 2024 19:07:12 +0300 Subject: [PATCH 10/15] Remove unwanted migration file Signed-off-by: ziadhany --- .../migrations/0063_exploit_delete_kev.py | 131 ------------------ 1 file changed, 131 deletions(-) delete mode 100644 vulnerabilities/migrations/0063_exploit_delete_kev.py diff --git a/vulnerabilities/migrations/0063_exploit_delete_kev.py b/vulnerabilities/migrations/0063_exploit_delete_kev.py deleted file mode 100644 index 00d2d60fe..000000000 --- a/vulnerabilities/migrations/0063_exploit_delete_kev.py +++ /dev/null @@ -1,131 +0,0 @@ -# Generated by Django 4.1.13 on 2024-09-10 18:40 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ("vulnerabilities", "0062_package_is_ghost"), - ] - - operations = [ - migrations.CreateModel( - name="Exploit", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, primary_key=True, serialize=False, verbose_name="ID" - ), - ), - ( - "date_added", - models.DateField( - blank=True, - help_text="The date the vulnerability was added to an exploit catalog.", - null=True, - ), - ), - ( - "description", - models.TextField( - blank=True, - help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", - null=True, - ), - ), - ( - "required_action", - models.TextField( - blank=True, - help_text="The required action to address the vulnerability, typically to apply vendor updates or apply vendor mitigations or to discontinue use.", - null=True, - ), - ), - ( - "due_date", - models.DateField( - blank=True, - help_text="The date the required action is due, which applies to all USA federal civilian executive branch (FCEB) agencies, but all organizations are strongly encouraged to execute the required action", - null=True, - ), - ), - ( - "notes", - models.TextField( - blank=True, - help_text="Additional notes and resources about the vulnerability, often a URL to vendor instructions.", - null=True, - ), - ), - ( - "known_ransomware_campaign_use", - models.BooleanField( - default=False, - help_text="Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; \n or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.", - ), - ), - ( - "source_date_published", - models.DateField( - blank=True, - help_text="The date that the exploit was published or disclosed.", - null=True, - ), - ), - ( - "exploit_type", - models.TextField( - blank=True, - help_text="The type of the exploit as provided by the original upstream data source.", - null=True, - ), - ), - ( - "platform", - models.TextField( - blank=True, - help_text="The platform associated with the exploit as provided by the original upstream data source.", - null=True, - ), - ), - ( - "source_date_updated", - models.DateField( - blank=True, - help_text="The date the exploit was updated in the original upstream data source.", - null=True, - ), - ), - ( - "data_source", - models.TextField( - blank=True, - help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", - null=True, - ), - ), - ( - "source_url", - models.URLField( - blank=True, - help_text="The URL to the exploit as provided in the original upstream data source.", - null=True, - ), - ), - ( - "vulnerability", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="exploits", - to="vulnerabilities.vulnerability", - ), - ), - ], - ), - migrations.DeleteModel( - name="Kev", - ), - ] From 3b965116fbd60db09be90d698e559a9c60d43f79 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 24 Sep 2024 05:09:40 +0300 Subject: [PATCH 11/15] Add log traceback for all the errors. Add missing logs Handle cases of one exploit for multiple vulnerabilities. Signed-off-by: ziadhany --- .../pipelines/enhance_with_exploitdb.py | 78 ++++++++++--------- vulnerabilities/pipelines/enhance_with_kev.py | 6 +- .../pipelines/enhance_with_metasploit.py | 43 +++++----- 3 files changed, 69 insertions(+), 58 deletions(-) diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py index eadf7919f..dcc3c986b 100644 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -1,6 +1,7 @@ import csv import io import logging +from traceback import format_exc as traceback_format_exc import requests from aboutcode.pipeline import LoopProgress @@ -34,13 +35,14 @@ def fetch_exploits(self): exploit_db_url = ( "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" ) + self.log(f"Fetching {exploit_db_url}") try: response = requests.get(exploit_db_url) response.raise_for_status() except requests.exceptions.HTTPError as http_err: self.log( - f"Failed to fetch the Exploit-DB Exploits: {exploit_db_url} - {http_err}", + f"Failed to fetch the Exploit-DB Exploits: {exploit_db_url} with error {http_err!r}:\n{traceback_format_exc()}", level=logging.ERROR, ) raise @@ -49,8 +51,7 @@ def fetch_exploits(self): def add_exploit(self): - csvreader = csv.reader(self.exploit_data) - header = next(csvreader) + csvreader = csv.DictReader(self.exploit_data) raw_data = list(csvreader) fetched_exploit_count = len(raw_data) @@ -60,55 +61,54 @@ def add_exploit(self): progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) for row in progress.iter(raw_data): - vulnerability_exploit_count += add_vulnerability_exploit(row, header, self.log) + vulnerability_exploit_count += add_vulnerability_exploit(row, self.log) self.log( f"Successfully added {vulnerability_exploit_count:,d} exploit-db vulnerability exploit" ) -def add_vulnerability_exploit(row, header, logger): - vulnerability = None - aliases = row[11].split(";") +def add_vulnerability_exploit(row, logger): + vulnerabilities = set() + aliases = row["codes"].split(";") for raw_alias in aliases: try: if alias := Alias.objects.get(alias=raw_alias): - vulnerability = alias.vulnerability - break + vulnerabilities.add(alias.vulnerability) except Alias.DoesNotExist: continue - if not vulnerability: + if not vulnerabilities: logger(f"No vulnerability found for aliases {aliases}") return 0 - add_exploit_references(row[11], row[16], row[1], vulnerability, logger) - - date_added = parse_date(row[header.index("date_added")]) - source_date_published = parse_date(row[header.index("date_published")]) - source_date_updated = parse_date(row[header.index("date_updated")]) - - try: - Exploit.objects.update_or_create( - vulnerability=vulnerability, - data_source="Exploit-DB", - defaults={ - "date_added": date_added, - "description": row[header.index("description")], - "known_ransomware_campaign_use": row[header.index("verified")], - "source_date_published": source_date_published, - "exploit_type": row[header.index("type")], - "platform": row[header.index("platform")], - "source_date_updated": source_date_updated, - "source_url": row[header.index("source_url")], - }, - ) - except DataError as e: - logger( - f"Failed to Create the Vulnerability Exploit-DB: {e}", - level=logging.ERROR, - ) + date_added = parse_date(row["date_added"]) + source_date_published = parse_date(row["date_published"]) + source_date_updated = parse_date(row["date_updated"]) + + for vulnerability in vulnerabilities: + add_exploit_references(row["codes"], row["source_url"], row["file"], vulnerability, logger) + try: + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="Exploit-DB", + defaults={ + "date_added": date_added, + "description": row["description"], + "known_ransomware_campaign_use": row["verified"], + "source_date_published": source_date_published, + "exploit_type": row["type"], + "platform": row["platform"], + "source_date_updated": source_date_updated, + "source_url": row["source_url"], + }, + ) + except DataError as e: + logger( + f"Failed to Create the Vulnerability Exploit-DB with error {e!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) return 1 @@ -137,7 +137,7 @@ def add_exploit_references(ref_id, direct_url, path, vul, logger): except DataError as e: logger( - f"Failed to Create the Vulnerability Reference For Exploit-DB: {e}", + f"Failed to Create the Vulnerability Reference For Exploit-DB with error {e!r}:\n{traceback_format_exc()}", level=logging.ERROR, ) @@ -148,5 +148,7 @@ def parse_date(date_string): date_obj = dateparser.parse(date_string).date() return date_obj.strftime("%Y-%m-%d") except (ValueError, TypeError, Exception) as e: - logging.error(f"Error while parsing ExploitDB date '{date_string}': {e}") + logging.error( + f"Error while parsing ExploitDB date '{date_string}' with error {e!r}:\n{traceback_format_exc()}" + ) return diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index 8ec801c5c..834d07320 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -1,4 +1,5 @@ import logging +from traceback import format_exc as traceback_format_exc import requests from aboutcode.pipeline import LoopProgress @@ -15,6 +16,7 @@ class VulnerabilityKevPipeline(VulnerableCodePipeline): """ pipeline_id = "enhance_with_kev" + license_expression = None @classmethod def steps(cls): @@ -27,12 +29,14 @@ def fetch_exploits(self): kev_url = ( "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" ) + self.log(f"Fetching {kev_url}") + try: response = requests.get(kev_url) response.raise_for_status() except requests.exceptions.HTTPError as http_err: self.log( - f"Failed to fetch the KEV Exploits: {kev_url} - {http_err}", + f"Failed to fetch the KEV Exploits: {kev_url} with error {http_err!r}:\n{traceback_format_exc()}", level=logging.ERROR, ) raise diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index 8bb2c79cc..3bc1ec5b3 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -1,4 +1,5 @@ import logging +from traceback import format_exc as traceback_format_exc import requests import saneyaml @@ -34,7 +35,8 @@ def fetch_exploits(self): response.raise_for_status() except requests.exceptions.HTTPError as http_err: self.log( - f"Failed to fetch the Metasploit Exploits: {url} - {http_err}", level=logging.ERROR + f"Failed to fetch the Metasploit Exploits: {url} with error {http_err!r}:\n{traceback_format_exc()}", + level=logging.ERROR, ) raise @@ -55,7 +57,7 @@ def add_vulnerability_exploits(self): def add_vulnerability_exploit(record, logger): - vulnerability = None + vulnerabilities = set() references = record.get("references", []) for ref in references: if ref.startswith("OSVDB") or ref.startswith("URL-"): @@ -64,12 +66,11 @@ def add_vulnerability_exploit(record, logger): try: if alias := Alias.objects.get(alias=ref): - vulnerability = alias.vulnerability - break + vulnerabilities.add(alias.vulnerability) except Alias.DoesNotExist: continue - if not vulnerability: + if not vulnerabilities: logger(f"No vulnerability found for aliases {references}") return 0 @@ -85,18 +86,22 @@ def add_vulnerability_exploit(record, logger): if disclosure_date := record.get("disclosure_date"): try: source_date_published = dateparser.parse(disclosure_date).date() - except ValueError: - logger(f"Error while parsing date {disclosure_date}", level=logging.ERROR) - - Exploit.objects.update_or_create( - vulnerability=vulnerability, - data_source="Metasploit", - defaults={ - "description": description, - "notes": saneyaml.dump(notes), - "source_date_published": source_date_published, - "platform": platform, - "source_url": source_url, - }, - ) + except ValueError as e: + logger( + f"Error while parsing date {disclosure_date} with error {e!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + + for vulnerability in vulnerabilities: + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="Metasploit", + defaults={ + "description": description, + "notes": saneyaml.dump(notes), + "source_date_published": source_date_published, + "platform": platform, + "source_url": source_url, + }, + ) return 1 From 5b3555209329187e567646d351357f437221cafa Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 24 Sep 2024 17:26:51 +0300 Subject: [PATCH 12/15] Skip empty aliases Remove empty vulnerability_kev.py file Signed-off-by: ziadhany --- vulnerabilities/improvers/vulnerability_kev.py | 0 vulnerabilities/pipelines/enhance_with_exploitdb.py | 6 +++++- vulnerabilities/pipelines/enhance_with_kev.py | 3 +++ vulnerabilities/pipelines/enhance_with_metasploit.py | 12 ++++++++---- 4 files changed, 16 insertions(+), 5 deletions(-) delete mode 100644 vulnerabilities/improvers/vulnerability_kev.py diff --git a/vulnerabilities/improvers/vulnerability_kev.py b/vulnerabilities/improvers/vulnerability_kev.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py index dcc3c986b..54554f951 100644 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -70,7 +70,11 @@ def add_exploit(self): def add_vulnerability_exploit(row, logger): vulnerabilities = set() - aliases = row["codes"].split(";") + + aliases = row["codes"].split(";") if row["codes"] else [] + + if not aliases: + return 0 for raw_alias in aliases: try: diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index 834d07320..6372bd3b0 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -61,6 +61,9 @@ def add_exploits(self): def add_vulnerability_exploit(kev_vul, logger): cve_id = kev_vul.get("cveID") + if not cve_id: + return 0 + vulnerability = None try: if alias := Alias.objects.get(alias=cve_id): diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index 3bc1ec5b3..1c2f8c825 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -59,11 +59,15 @@ def add_vulnerability_exploits(self): def add_vulnerability_exploit(record, logger): vulnerabilities = set() references = record.get("references", []) - for ref in references: - if ref.startswith("OSVDB") or ref.startswith("URL-"): - # ignore OSV-DB and reference exploit for metasploit - continue + interesting_references = [ + ref for ref in references if not ref.startswith("OSVDB") and not ref.startswith("URL-") + ] + + if not interesting_references: + return 0 + + for ref in interesting_references: try: if alias := Alias.objects.get(alias=ref): vulnerabilities.add(alias.vulnerability) From ca853fa6e9fd8c77825728afce1347f1d096aa92 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 24 Sep 2024 18:49:27 +0300 Subject: [PATCH 13/15] Replace references log with interesting_references Signed-off-by: ziadhany --- vulnerabilities/pipelines/enhance_with_metasploit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index 1c2f8c825..72897abd0 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -75,7 +75,7 @@ def add_vulnerability_exploit(record, logger): continue if not vulnerabilities: - logger(f"No vulnerability found for aliases {references}") + logger(f"No vulnerability found for aliases {interesting_references}") return 0 description = record.get("description", "") From 1b789563c023dfd190004d910291d0da2a122fde Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 30 Sep 2024 17:10:44 +0530 Subject: [PATCH 14/15] Use proper labels in vulnerability details Signed-off-by: Keshav Priyadarshi --- .../templates/vulnerability_details.html | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 450276db6..7350cc9c9 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -415,7 +415,7 @@ + data-tooltip="The source of the exploit information, such as CISA KEV, Exploit-DB, Metasploit, Packet Storm, or others."> Data source @@ -425,7 +425,7 @@ @@ -436,7 +436,7 @@ @@ -448,7 +448,7 @@ - required_action: + Required action @@ -461,7 +461,7 @@ data-tooltip="The date the required action is due in the format YYYY-MM-DD, which applies to all USA federal civilian executive branch (FCEB) agencies, but all organizations are strongly encouraged to execute the required action."> - due_date: + Due date @@ -473,7 +473,7 @@ - notes: + Note @@ -484,7 +484,7 @@ @@ -495,7 +495,7 @@ @@ -506,7 +506,7 @@ @@ -517,7 +517,7 @@ @@ -529,7 +529,7 @@ - source_date_updated: + Source update date @@ -542,7 +542,7 @@ - source_url: + Source URL @@ -572,7 +572,7 @@ @@ -582,7 +582,7 @@ @@ -594,7 +594,7 @@ - Published at: + Published at From d9e00db11213301d71dd6f1276d7292ef71f76b4 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 30 Sep 2024 17:25:29 +0530 Subject: [PATCH 15/15] Display Known/Unknown for ransomware campaign use Signed-off-by: Keshav Priyadarshi --- vulnerabilities/templates/vulnerability_details.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 7350cc9c9..d1f2fb6de 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -479,7 +479,7 @@ {% endif %} - {% if exploit.known_ransomware_campaign_use %} + {% if exploit.known_ransomware_campaign_use is not None %} - + {% endif %} {% if exploit.source_date_published %}
    data_source {{ exploit.data_source }}
    - date_added: + Date added {{ exploit.date_added }} - Description: + Description {{ exploit.description }} {{ exploit.required_action }} {{ exploit.due_date }} 
    {{ exploit.notes }}
    		 - known_ransomware_campaign_use: + Ransomware campaign use {{ exploit.known_ransomware_campaign_use }} - source_date_published: + Source publication date {{ exploit.source_date_published }} - exploit_type: + Exploit type {{ exploit.exploit_type }} - platform: + Platform {{ exploit.platform }} {{ exploit.source_date_updated }} {{ exploit.source_url }} - Percentile: + Percentile {{ severity.scoring_elements }} - EPSS score: + EPSS score {{ severity.value }} {{ severity.published_at }}
    {{ exploit.notes }}
    {{ exploit.known_ransomware_campaign_use }}{{ exploit.known_ransomware_campaign_use|yesno:"Known,Unknown" }}