From 4e90f8da1ef5ded689badf125c52471294b89661 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 17 Jan 2023 17:20:50 +0200 Subject: [PATCH] Add ruby importer Signed-off-by: ziadhany --- vulnerabilities/importers/__init__.py | 2 + vulnerabilities/importers/ruby.py | 235 ++++++++++++++++---------- vulnerabilities/improvers/__init__.py | 1 + 3 files changed, 145 insertions(+), 93 deletions(-) diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 5f4957421..553a25b0a 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -26,6 +26,7 @@ from vulnerabilities.importers import pysec from vulnerabilities.importers import redhat from vulnerabilities.importers import retiredotnet +from vulnerabilities.importers import ruby from vulnerabilities.importers import ubuntu IMPORTERS_REGISTRY = [ @@ -49,6 +50,7 @@ mozilla.MozillaImporter, gentoo.GentooImporter, istio.IstioImporter, + ruby.RubyImporter, ] IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py index 2c1b542b7..ff18b10f8 100644 --- a/vulnerabilities/importers/ruby.py +++ b/vulnerabilities/importers/ruby.py @@ -8,10 +8,12 @@ # import logging +from pathlib import Path from typing import Iterable from typing import List from dateutil.parser import parse +from django.db.models import QuerySet from packageurl import PackageURL from pytz import UTC from univers.version_range import GemVersionRange @@ -19,10 +21,14 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import GitConfig from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference +from vulnerabilities.importer import UnMergeablePackageError from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.improver import MAX_CONFIDENCE +from vulnerabilities.improver import Improver +from vulnerabilities.improver import Inference +from vulnerabilities.models import Advisory from vulnerabilities.package_managers import RubyVersionAPI from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import build_description @@ -33,75 +39,69 @@ class RubyImporter(GitImporter): license_url = "https://github.com/rubysec/ruby-advisory-db/blob/master/LICENSE.txt" - spdx_license_expression = "unknow" - config = GitConfig( - repository_url="https://github.com/rubysec/ruby-advisory-db.git", - working_directory="ruby", - branch="master", - ) - cutoff_timestamp = 1 + spdx_license_expression = "unknown" def __init__(self): - super().__init__(config=self.config, cutoff_timestamp=self.cutoff_timestamp) - self.pkg_manager_api = RubyVersionAPI() + super().__init__(repo_url="git+https://github.com/rubysec/ruby-advisory-db") - self._added_files_gems, self._updated_files_gems = self.file_changes( - recursive=True, - file_ext="yml", - subdir="./gems", - ) - - self._added_files_rubies, self._updated_files_rubies = self.file_changes( - recursive=True, file_ext="yml", subdir="./rubies" - ) - - def parse_ruby_advisory(self, record, schema_type): - """ - Parse a ruby advisory file and return an AdvisoryData or None. - Each advisory file contains the advisory information in YAML format. - Schema: https://github.com/rubysec/ruby-advisory-db/tree/master/spec/schemas - """ - if schema_type == "gem": - package_name = record.get("gem") - library = record.get("library") - framework = record.get("framework") - platform = record.get("platform") - purl = PackageURL(type="gem", name=package_name) - safe_version_ranges = record.get("patched_versions", []) - # this case happens when the advisory contain only 'patched_versions' field - # and it has value None(i.e it is empty :( ). - if not safe_version_ranges: - safe_version_ranges = [] - safe_version_ranges += record.get("unaffected_versions", []) - safe_version_ranges = [i for i in safe_version_ranges if i] - all_vers = self.pkg_manager_api.fetch(package_name) - - affected_packages = get_aff_pkg(purl, all_vers, safe_version_ranges) - return AdvisoryData( - aliases=get_aliases(record), - summary=get_summary(record), - affected_packages=affected_packages, - references=get_references(record), - date_published=get_publish_time(record), - ) - elif schema_type == "ruby": - return AdvisoryData( - aliases=get_aliases(record), - summary=get_summary(record), - references=get_references(record), - date_published=get_publish_time(record), + def advisory_data(self) -> Iterable[AdvisoryData]: + self.clone() + base_path = Path(self.vcs_response.dest_dir) + supported_subdir = ["rubies", "gems"] + for subdir in supported_subdir: + for file_path in base_path.glob(f"{subdir}/**/*.yml"): + raw_data = load_yaml(file_path) + yield parse_ruby_advisory(raw_data, subdir) + + +def parse_ruby_advisory(record, schema_type): + """ + Parse a ruby advisory file and return an AdvisoryData or None. + Each advisory file contains the advisory information in YAML format. + Schema: https://github.com/rubysec/ruby-advisory-db/tree/master/spec/schemas + """ + if schema_type == "gems": + package_name = record.get("gem") + library = record.get("library") + framework = record.get("framework") + platform = record.get("platform") + purl = PackageURL(type="gem", name=package_name) + safe_version_ranges = record.get("patched_versions", []) + # this case happens when the advisory contain only 'patched_versions' field + # and it has value None(i.e it is empty :( ). + if not safe_version_ranges: + safe_version_ranges = [] + safe_version_ranges += record.get("unaffected_versions", []) + safe_version_ranges = [i for i in safe_version_ranges if i] + + affected_packages = [] + affected_version_ranges = [ + GemVersionRange.from_native(elem).invert() for elem in safe_version_ranges + ] + + for affected_version_range in affected_version_ranges: + affected_packages.append( + AffectedPackage( + package=purl, + affected_version_range=affected_version_range, + ) ) - def advisory_data(self) -> Iterable[AdvisoryData]: - files = self._updated_files_gems.union(self._added_files_gems) - for file in files: - raw_data = load_yaml(file) - yield self.parse_ruby_advisory(raw_data, schema_type="gem") + return AdvisoryData( + aliases=get_aliases(record), + summary=get_summary(record), + affected_packages=affected_packages, + references=get_references(record), + date_published=get_publish_time(record), + ) - files = self._added_files_rubies.union(self._updated_files_rubies) - for file in files: - raw_data = load_yaml(file) - yield self.parse_ruby_advisory(raw_data, schema_type="ruby") + elif schema_type == "rubies": + return AdvisoryData( + aliases=get_aliases(record), + summary=get_summary(record), + references=get_references(record), + date_published=get_publish_time(record), + ) def get_aliases(record) -> [str]: @@ -154,49 +154,98 @@ def get_summary(record): return build_description(summary=title, description=description) -def get_aff_pkg(purl, all_vers, safe_version_ranges) -> List[AffectedPackage]: - affected_packages = [] - fixed_versions, affected_versions = categorize_versions(all_vers, safe_version_ranges) +class RubyImprover(Improver): + pkg_manager_api = RubyVersionAPI() - affected_version = [vers for vers in all_vers if vers not in safe_version_ranges] - affected_version_range = GemVersionRange.from_versions(affected_version) - for fixed_version in fixed_versions: - affected_packages.append( - AffectedPackage( - package=purl, - affected_version_range=affected_version_range, - fixed_version=fixed_version, - ) - ) - return affected_packages + # affected_packages = get_aff_pkg(purl, all_vers, safe_version_ranges) + @property + def interesting_advisories(self) -> QuerySet: + return Advisory.objects.filter(created_by=RubyImporter.qualified_name) + def get_inferences(self, advisory_data) -> Iterable[Inference]: + try: + purl, affected_version_ranges, _ = AffectedPackage.merge( + advisory_data.affected_packages + ) + except UnMergeablePackageError: + logger.error(f"Cannot merge with different purls {advisory_data.affected_packages!r}") + return iter([]) -def categorize_versions(all_versions, unaffected_version_ranges): - try: - for id, elem in enumerate(unaffected_version_ranges): - try: - unaffected_version_ranges[id] = GemVersionRange.from_native(elem) - except: - logger.error(f"Invalid VersionRange {elem}") + pkg_name = purl.name + all_vers_pkgs = self.pkg_manager_api.fetch(pkg_name) safe_versions = [] vulnerable_versions = [] - for i in all_versions: + for i in all_vers_pkgs: vobj = RubygemsVersion(i.value) is_vulnerable = False try: - for ver_rng in unaffected_version_ranges: - if vobj in ver_rng: + for ver_rng in affected_version_ranges: + if vobj not in ver_rng: safe_versions.append(i.value) is_vulnerable = True break + if not is_vulnerable: vulnerable_versions.append(i.value) except Exception as e: logger.error(f"{e}") - return safe_versions, vulnerable_versions - except: - logger.error( - f"Invalid Version affected_pkg Error {all_versions} , {unaffected_version_ranges} " + + affected_purls = [] + fixed_purl = None + yield Inference( + affected_purls=affected_purls, + # fixed_purl=fixed_purl, + confidence=MAX_CONFIDENCE, ) - return [], [] + + +# def get_aff_pkg(purl, all_vers, safe_version_ranges) -> List[AffectedPackage]: +# """ +# +# """ +# affected_packages = [] +# fixed_versions, affected_versions = categorize_versions(all_vers, safe_version_ranges) +# +# affected_version = [vers for vers in all_vers if vers not in safe_version_ranges] +# affected_version_range = GemVersionRange.from_versions(affected_version) +# for fixed_version in fixed_versions: +# affected_packages.append( +# AffectedPackage( +# package=purl, +# affected_version_range=affected_version_range, +# fixed_version=fixed_version, +# ) +# ) +# return affected_packages +# +# +# def categorize_versions(all_versions, unaffected_version_ranges): +# try: +# for id, elem in enumerate(unaffected_version_ranges): +# try: +# unaffected_version_ranges[id] = GemVersionRange.from_native(elem) +# except: +# logger.error(f"Invalid VersionRange {elem}") +# +# safe_versions = [] +# vulnerable_versions = [] +# for i in all_versions: +# vobj = RubygemsVersion(i.value) +# is_vulnerable = False +# try: +# for ver_rng in unaffected_version_ranges: +# if vobj in ver_rng: +# safe_versions.append(i.value) +# is_vulnerable = True +# break +# if not is_vulnerable: +# vulnerable_versions.append(i.value) +# except Exception as e: +# logger.error(f"{e}") +# return safe_versions, vulnerable_versions +# except: +# logger.error( +# f"Invalid Version affected_pkg Error {all_versions} , {unaffected_version_ranges} " +# ) +# return [], [] diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 98192e264..0ba520cb7 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -19,6 +19,7 @@ importers.gitlab.GitLabBasicImprover, oval.DebianOvalBasicImprover, oval.UbuntuOvalBasicImprover, + importers.ruby.RubyImprover, ] IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY}