Skip to content

Commit

Permalink
Add ruby importer
Browse files Browse the repository at this point in the history 
Signed-off-by: ziadhany <ziadhany2016@gmail.com>
  • Loading branch information
@ziadhany
ziadhany committed Jan 17, 2023
1 parent 9303d92 commit 4e90f8d
Show file tree
Hide file tree
Showing 3 changed files with 145 additions and 93 deletions.
2 changes: 2 additions & 0 deletions vulnerabilities/importers/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from vulnerabilities.importers import pysec
from vulnerabilities.importers import redhat
from vulnerabilities.importers import retiredotnet
from vulnerabilities.importers import ruby
from vulnerabilities.importers import ubuntu

IMPORTERS_REGISTRY = [
Expand All @@ -49,6 +50,7 @@
mozilla.MozillaImporter,
gentoo.GentooImporter,
istio.IstioImporter,
ruby.RubyImporter,
]

IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}
235 changes: 142 additions & 93 deletions vulnerabilities/importers/ruby.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,27 @@
#

import logging
from pathlib import Path
from typing import Iterable
from typing import List

from dateutil.parser import parse
from django.db.models import QuerySet
from packageurl import PackageURL
from pytz import UTC
from univers.version_range import GemVersionRange
from univers.versions import RubygemsVersion

from vulnerabilities.importer import AdvisoryData
from vulnerabilities.importer import AffectedPackage
from vulnerabilities.importer import GitConfig
from vulnerabilities.importer import GitImporter
from vulnerabilities.importer import Reference
from vulnerabilities.importer import UnMergeablePackageError
from vulnerabilities.importer import VulnerabilitySeverity
from vulnerabilities.improver import MAX_CONFIDENCE
from vulnerabilities.improver import Improver
from vulnerabilities.improver import Inference
from vulnerabilities.models import Advisory
from vulnerabilities.package_managers import RubyVersionAPI
from vulnerabilities.severity_systems import SCORING_SYSTEMS
from vulnerabilities.utils import build_description
Expand All @@ -33,75 +39,69 @@

class RubyImporter(GitImporter):
license_url = "https://github.com/rubysec/ruby-advisory-db/blob/master/LICENSE.txt"
spdx_license_expression = "unknow"
config = GitConfig(
repository_url="https://github.com/rubysec/ruby-advisory-db.git",
working_directory="ruby",
branch="master",
)
cutoff_timestamp = 1
spdx_license_expression = "unknown"

def __init__(self):
super().__init__(config=self.config, cutoff_timestamp=self.cutoff_timestamp)
self.pkg_manager_api = RubyVersionAPI()
super().__init__(repo_url="git+https://github.com/rubysec/ruby-advisory-db")

self._added_files_gems, self._updated_files_gems = self.file_changes(
recursive=True,
file_ext="yml",
subdir="./gems",
)

self._added_files_rubies, self._updated_files_rubies = self.file_changes(
recursive=True, file_ext="yml", subdir="./rubies"
)

def parse_ruby_advisory(self, record, schema_type):
"""
Parse a ruby advisory file and return an AdvisoryData or None.
Each advisory file contains the advisory information in YAML format.
Schema: https://github.com/rubysec/ruby-advisory-db/tree/master/spec/schemas
"""
if schema_type == "gem":
package_name = record.get("gem")
library = record.get("library")
framework = record.get("framework")
platform = record.get("platform")
purl = PackageURL(type="gem", name=package_name)
safe_version_ranges = record.get("patched_versions", [])
# this case happens when the advisory contain only 'patched_versions' field
# and it has value None(i.e it is empty :( ).
if not safe_version_ranges:
safe_version_ranges = []
safe_version_ranges += record.get("unaffected_versions", [])
safe_version_ranges = [i for i in safe_version_ranges if i]
all_vers = self.pkg_manager_api.fetch(package_name)

affected_packages = get_aff_pkg(purl, all_vers, safe_version_ranges)
return AdvisoryData(
aliases=get_aliases(record),
summary=get_summary(record),
affected_packages=affected_packages,
references=get_references(record),
date_published=get_publish_time(record),
)
elif schema_type == "ruby":
return AdvisoryData(
aliases=get_aliases(record),
summary=get_summary(record),
references=get_references(record),
date_published=get_publish_time(record),
def advisory_data(self) -> Iterable[AdvisoryData]:
self.clone()
base_path = Path(self.vcs_response.dest_dir)
supported_subdir = ["rubies", "gems"]
for subdir in supported_subdir:
for file_path in base_path.glob(f"{subdir}/**/*.yml"):
raw_data = load_yaml(file_path)
yield parse_ruby_advisory(raw_data, subdir)


def parse_ruby_advisory(record, schema_type):
"""
Parse a ruby advisory file and return an AdvisoryData or None.
Each advisory file contains the advisory information in YAML format.
Schema: https://github.com/rubysec/ruby-advisory-db/tree/master/spec/schemas
"""
if schema_type == "gems":
package_name = record.get("gem")
library = record.get("library")
framework = record.get("framework")
platform = record.get("platform")
purl = PackageURL(type="gem", name=package_name)
safe_version_ranges = record.get("patched_versions", [])
# this case happens when the advisory contain only 'patched_versions' field
# and it has value None(i.e it is empty :( ).
if not safe_version_ranges:
safe_version_ranges = []
safe_version_ranges += record.get("unaffected_versions", [])
safe_version_ranges = [i for i in safe_version_ranges if i]

affected_packages = []
affected_version_ranges = [
GemVersionRange.from_native(elem).invert() for elem in safe_version_ranges
]

for affected_version_range in affected_version_ranges:
affected_packages.append(
AffectedPackage(
package=purl,
affected_version_range=affected_version_range,
)
)

def advisory_data(self) -> Iterable[AdvisoryData]:
files = self._updated_files_gems.union(self._added_files_gems)
for file in files:
raw_data = load_yaml(file)
yield self.parse_ruby_advisory(raw_data, schema_type="gem")
return AdvisoryData(
aliases=get_aliases(record),
summary=get_summary(record),
affected_packages=affected_packages,
references=get_references(record),
date_published=get_publish_time(record),
)

files = self._added_files_rubies.union(self._updated_files_rubies)
for file in files:
raw_data = load_yaml(file)
yield self.parse_ruby_advisory(raw_data, schema_type="ruby")
elif schema_type == "rubies":
return AdvisoryData(
aliases=get_aliases(record),
summary=get_summary(record),
references=get_references(record),
date_published=get_publish_time(record),
)


def get_aliases(record) -> [str]:
Expand Down Expand Up @@ -154,49 +154,98 @@ def get_summary(record):
return build_description(summary=title, description=description)


def get_aff_pkg(purl, all_vers, safe_version_ranges) -> List[AffectedPackage]:
affected_packages = []
fixed_versions, affected_versions = categorize_versions(all_vers, safe_version_ranges)
class RubyImprover(Improver):
pkg_manager_api = RubyVersionAPI()

affected_version = [vers for vers in all_vers if vers not in safe_version_ranges]
affected_version_range = GemVersionRange.from_versions(affected_version)
for fixed_version in fixed_versions:
affected_packages.append(
AffectedPackage(
package=purl,
affected_version_range=affected_version_range,
fixed_version=fixed_version,
)
)
return affected_packages
# affected_packages = get_aff_pkg(purl, all_vers, safe_version_ranges)
@property
def interesting_advisories(self) -> QuerySet:
return Advisory.objects.filter(created_by=RubyImporter.qualified_name)

def get_inferences(self, advisory_data) -> Iterable[Inference]:
try:
purl, affected_version_ranges, _ = AffectedPackage.merge(
advisory_data.affected_packages
)
except UnMergeablePackageError:
logger.error(f"Cannot merge with different purls {advisory_data.affected_packages!r}")
return iter([])

def categorize_versions(all_versions, unaffected_version_ranges):
try:
for id, elem in enumerate(unaffected_version_ranges):
try:
unaffected_version_ranges[id] = GemVersionRange.from_native(elem)
except:
logger.error(f"Invalid VersionRange {elem}")
pkg_name = purl.name
all_vers_pkgs = self.pkg_manager_api.fetch(pkg_name)

safe_versions = []
vulnerable_versions = []
for i in all_versions:
for i in all_vers_pkgs:
vobj = RubygemsVersion(i.value)
is_vulnerable = False
try:
for ver_rng in unaffected_version_ranges:
if vobj in ver_rng:
for ver_rng in affected_version_ranges:
if vobj not in ver_rng:
safe_versions.append(i.value)
is_vulnerable = True
break

if not is_vulnerable:
vulnerable_versions.append(i.value)
except Exception as e:
logger.error(f"{e}")
return safe_versions, vulnerable_versions
except:
logger.error(
f"Invalid Version affected_pkg Error {all_versions} , {unaffected_version_ranges} "

affected_purls = []
fixed_purl = None
yield Inference(
affected_purls=affected_purls,
# fixed_purl=fixed_purl,
confidence=MAX_CONFIDENCE,
)
return [], []


# def get_aff_pkg(purl, all_vers, safe_version_ranges) -> List[AffectedPackage]:
# """
#
# """
# affected_packages = []
# fixed_versions, affected_versions = categorize_versions(all_vers, safe_version_ranges)
#
# affected_version = [vers for vers in all_vers if vers not in safe_version_ranges]
# affected_version_range = GemVersionRange.from_versions(affected_version)
# for fixed_version in fixed_versions:
# affected_packages.append(
# AffectedPackage(
# package=purl,
# affected_version_range=affected_version_range,
# fixed_version=fixed_version,
# )
# )
# return affected_packages
#
#
# def categorize_versions(all_versions, unaffected_version_ranges):
# try:
# for id, elem in enumerate(unaffected_version_ranges):
# try:
# unaffected_version_ranges[id] = GemVersionRange.from_native(elem)
# except:
# logger.error(f"Invalid VersionRange {elem}")
#
# safe_versions = []
# vulnerable_versions = []
# for i in all_versions:
# vobj = RubygemsVersion(i.value)
# is_vulnerable = False
# try:
# for ver_rng in unaffected_version_ranges:
# if vobj in ver_rng:
# safe_versions.append(i.value)
# is_vulnerable = True
# break
# if not is_vulnerable:
# vulnerable_versions.append(i.value)
# except Exception as e:
# logger.error(f"{e}")
# return safe_versions, vulnerable_versions
# except:
# logger.error(
# f"Invalid Version affected_pkg Error {all_versions} , {unaffected_version_ranges} "
# )
# return [], []
1 change: 1 addition & 0 deletions vulnerabilities/improvers/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
importers.gitlab.GitLabBasicImprover,
oval.DebianOvalBasicImprover,
oval.UbuntuOvalBasicImprover,
importers.ruby.RubyImprover,
]

IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY}

0 comments on commit 4e90f8d

Please sign in to comment.