Merge branch 'main' into 1079-create-suse-oval-importer #1079

Reference: #1079

Signed-off-by: John M. Horan <johnmhoran@gmail.com>

CHANGELOG.rst

@@ -2,6 +2,24 @@ Release notes
 =============
 
 
+Version v33.6.5
+-------------------
+
+- We added /var/www/html as volume in nginx Docker compose (#1373).
+
+
+Version v33.6.4
+-------------------
+
+- We added /var/www/html as volume in Docker compose (#1371).
+- We fixed table borders in Vulnerability details UI #1356 (#1358)
+- We fixed import runner's process_inferences (#1360)
+- We fixed debian OVAL importer (#1361)
+- We added graph model diagrams #977(#1350)
+- We added endpoint for purl lookup (#1359)
+- We fixed swagger API docs generation (#1366)
+
+
 Version v33.6.3
 ----------------
 

docker-compose.yml

@@ -34,6 +34,7 @@ services:
     volumes:
       - ./etc/nginx/conf.d/:/etc/nginx/conf.d/
       - static:/var/vulnerablecode/static/
+      - /var/www/html:/var/www/html
     depends_on:
       - vulnerablecode
 

requirements.txt

@@ -55,7 +55,7 @@ multidict==6.0.2
 mypy-extensions==0.4.3
 packageurl-python==0.10.5rc1
 packaging==21.3
-paramiko==2.10.3
+paramiko==3.4.0
 parso==0.8.3
 pathspec==0.9.0
 pbr==5.8.1
@@ -113,7 +113,7 @@ websocket-client==0.59.0
 yarl==1.7.2
 zipp==3.8.0
 dateparser==1.1.1
-fetchcode==0.2.0
+fetchcode==0.3.0
 cwe2==2.0.0
 drf-spectacular-sidecar==2022.10.1
 drf-spectacular==0.24.2

setup.cfg

@@ -90,7 +90,7 @@ install_requires =
     # networking
     GitPython>=3.1.17
     requests>=2.25.1
-    fetchcode>=0.2.0
+    fetchcode>=0.3.0
 
     #vulntotal
     python-dotenv

vulnerabilities/api.py

@@ -11,8 +11,11 @@
 
 from django.db.models import Prefetch
 from django_filters import rest_framework as filters
+from drf_spectacular.utils import extend_schema
+from drf_spectacular.utils import inline_serializer
 from packageurl import PackageURL
 from rest_framework import serializers
+from rest_framework import status
 from rest_framework import viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
@@ -272,6 +275,26 @@ def filter_purl(self, queryset, name, value):
         return self.queryset.filter(**lookups)
 
 
+class PackageurlListSerializer(serializers.Serializer):
+    purls = serializers.ListField(
+        child=serializers.CharField(),
+        allow_empty=False,
+        help_text="List of PackageURL strings in canonical form.",
+    )
+
+
+class PackageBulkSearchRequestSerializer(PackageurlListSerializer):
+    purl_only = serializers.BooleanField(required=False, default=False)
+    plain_purl = serializers.BooleanField(required=False, default=False)
+
+
+class LookupRequestSerializer(serializers.Serializer):
+    purl = serializers.CharField(
+        required=True,
+        help_text="PackageURL strings in canonical form.",
+    )
+
+
 class PackageViewSet(viewsets.ReadOnlyModelViewSet):
     """
     Lookup for vulnerable packages by Package URL.
@@ -283,21 +306,34 @@ class PackageViewSet(viewsets.ReadOnlyModelViewSet):
     filterset_class = PackageFilterSet
     throttle_classes = [StaffUserRateThrottle, AnonRateThrottle]
 
-    # TODO: Fix the swagger documentation for this endpoint
-    @action(detail=False, methods=["post"])
+    @extend_schema(
+        request=PackageBulkSearchRequestSerializer,
+        responses={200: PackageSerializer(many=True)},
+    )
+    @action(
+        detail=False,
+        methods=["post"],
+        serializer_class=PackageBulkSearchRequestSerializer,
+        filter_backends=[],
+        pagination_class=None,
+    )
     def bulk_search(self, request):
         """
         Lookup for vulnerable packages using many Package URLs at once.
         """
-
-        purls = request.data.get("purls", []) or []
-        purl_only = request.data.get("purl_only", False)
-        plain_purl = request.data.get("plain_purl", False)
-        if not purls or not isinstance(purls, list):
+        serializer = self.serializer_class(data=request.data)
+        if not serializer.is_valid():
             return Response(
-                status=400,
-                data={"Error": "A non-empty 'purls' list of PURLs is required."},
+                status=status.HTTP_400_BAD_REQUEST,
+                data={
+                    "error": serializer.errors,
+                    "message": "A non-empty 'purls' list of PURLs is required.",
+                },
             )
+        validated_data = serializer.validated_data
+        purls = validated_data.get("purls")
+        purl_only = validated_data.get("purl_only", False)
+        plain_purl = validated_data.get("plain_purl", False)
 
         if plain_purl:
             purl_objects = [PackageURL.from_string(purl) for purl in purls]
@@ -347,6 +383,74 @@ def all(self, request):
         vulnerable_purls = [str(package.package_url) for package in vulnerable_packages]
         return Response(vulnerable_purls)
 
+    @extend_schema(
+        request=LookupRequestSerializer,
+        responses={200: PackageSerializer(many=True)},
+    )
+    @action(
+        detail=False,
+        methods=["post"],
+        serializer_class=LookupRequestSerializer,
+        filter_backends=[],
+        pagination_class=None,
+    )
+    def lookup(self, request):
+        """
+        Return the response for exact PackageURL requested for.
+        """
+        serializer = self.serializer_class(data=request.data)
+        if not serializer.is_valid():
+            return Response(
+                status=status.HTTP_400_BAD_REQUEST,
+                data={
+                    "error": serializer.errors,
+                    "message": "A 'purl' is required.",
+                },
+            )
+        validated_data = serializer.validated_data
+        purl = validated_data.get("purl")
+
+        return Response(
+            PackageSerializer(
+                Package.objects.for_purls([purl]), many=True, context={"request": request}
+            ).data
+        )
+
+    @extend_schema(
+        request=PackageurlListSerializer,
+        responses={200: PackageSerializer(many=True)},
+    )
+    @action(
+        detail=False,
+        methods=["post"],
+        serializer_class=PackageurlListSerializer,
+        filter_backends=[],
+        pagination_class=None,
+    )
+    def bulk_lookup(self, request):
+        """
+        Return the response for exact PackageURLs requested for.
+        """
+        serializer = self.serializer_class(data=request.data)
+        if not serializer.is_valid():
+            return Response(
+                status=status.HTTP_400_BAD_REQUEST,
+                data={
+                    "error": serializer.errors,
+                    "message": "A non-empty 'purls' list of PURLs is required.",
+                },
+            )
+        validated_data = serializer.validated_data
+        purls = validated_data.get("purls")
+
+        return Response(
+            PackageSerializer(
+                Package.objects.for_purls(purls),
+                many=True,
+                context={"request": request},
+            ).data
+        )
+
 
 class VulnerabilityFilterSet(filters.FilterSet):
     class Meta:

vulnerabilities/importers/istio.py

@@ -34,8 +34,6 @@
 from vulnerabilities.improver import Improver
 from vulnerabilities.improver import Inference
 from vulnerabilities.models import Advisory
-from vulnerabilities.package_managers import GitHubTagsAPI
-from vulnerabilities.package_managers import VersionAPI
 from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage
 from vulnerabilities.utils import get_affected_packages_by_patched_package
 from vulnerabilities.utils import nearest_patched_package

vulnerabilities/improvers/valid_versions.py

@@ -17,6 +17,7 @@
 
 from django.db.models import Q
 from django.db.models.query import QuerySet
+from fetchcode import package_versions
 from packageurl import PackageURL
 from univers.versions import NginxVersion
 
@@ -41,12 +42,6 @@
 from vulnerabilities.improver import Improver
 from vulnerabilities.improver import Inference
 from vulnerabilities.models import Advisory
-from vulnerabilities.package_managers import GitHubTagsAPI
-from vulnerabilities.package_managers import GoproxyVersionAPI
-from vulnerabilities.package_managers import PackageVersion
-from vulnerabilities.package_managers import VersionAPI
-from vulnerabilities.package_managers import get_api_package_name
-from vulnerabilities.package_managers import get_version_fetcher
 from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage
 from vulnerabilities.utils import clean_nginx_git_tag
 from vulnerabilities.utils import evolve_purl
@@ -58,14 +53,11 @@
 logger = logging.getLogger(__name__)
 
 
-@dataclasses.dataclass(order=True)
+@dataclasses.dataclass(order=True, init=False)
 class ValidVersionImprover(Improver):
     importer: Importer
     ignorable_versions: List[str] = dataclasses.field(default_factory=list)
 
-    def __init__(self) -> None:
-        self.versions_fetcher_by_purl: Mapping[str, VersionAPI] = {}
-
     @property
     def interesting_advisories(self) -> QuerySet:
         return Advisory.objects.filter(Q(created_by=self.importer.qualified_name)).paginated()
@@ -74,21 +66,16 @@ def get_package_versions(
         self, package_url: PackageURL, until: Optional[datetime] = None
     ) -> List[str]:
         """
-        Return a list of `valid_versions` for the `package_url`
+        Return a list of versions published before `until` for the `package_url`
         """
-        api_name = get_api_package_name(package_url)
-        if not api_name:
-            logger.error(f"Could not get versions for {package_url!r}")
-            return []
-        versions_fetcher = self.versions_fetcher_by_purl.get(package_url)
-        if not versions_fetcher:
-            versions_fetcher = get_version_fetcher(package_url)
-            self.versions_fetcher_by_purl[package_url] = versions_fetcher()
-
-        versions_fetcher = self.versions_fetcher_by_purl[package_url]
+        versions = package_versions.versions(str(package_url))
+        versions_before_until = []
+        for version in versions or []:
+            if until and version.release_date and version.release_date > until:
+                continue
+            versions_before_until.append(version.value)
 
-        self.versions_fetcher_by_purl[package_url] = versions_fetcher
-        return versions_fetcher.get_until(package_name=api_name, until=until).valid_versions
+        return versions_before_until
 
     def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
         """
@@ -163,15 +150,6 @@ def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
                         fixed_purl=fixed_purl,
                     )
             else:
-                if purl.type == "golang":
-                    # Problem with the Golang and Go that they provide full path
-                    # FIXME: We need to get the PURL subpath for Go module
-                    versions_fetcher = self.versions_fetcher_by_purl.get(purl)
-                    if not versions_fetcher:
-                        versions_fetcher = GoproxyVersionAPI()
-                        self.versions_fetcher_by_purl[purl] = versions_fetcher
-                    pkg_name = versions_fetcher.module_name_by_package_name.get(pkg_name, pkg_name)
-
                 valid_versions = self.get_package_versions(
                     package_url=purl, until=advisory_data.date_published
                 )
@@ -248,11 +226,10 @@ def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
         )
 
     def get_inferences_from_versions(
-        self, advisory_data: AdvisoryData, all_versions: List[PackageVersion]
+        self, advisory_data: AdvisoryData, all_versions: List[str]
     ) -> Iterable[Inference]:
         """
-        Yield inferences given an ``advisory_data`` and a ``all_versions`` of
-        PackageVersion.
+        Yield inferences given an ``advisory_data`` and a ``all_versions``.
         """
 
         try:
@@ -268,9 +245,9 @@ def get_inferences_from_versions(
 
         affected_purls = []
         for affected_version_range in affected_version_ranges:
-            for package_version in all_versions:
+            for version in all_versions:
                 # FIXME: we should reference an NginxVersion tbd in univers
-                version = NginxVersion(package_version.value)
+                version = NginxVersion(version)
                 if is_vulnerable_nginx_version(
                     version=version,
                     affected_version_range=affected_version_range,
@@ -294,12 +271,12 @@ def get_inferences_from_versions(
 
     def fetch_nginx_version_from_git_tags(self):
         """
-        Yield all nginx PackageVersion from its git tags.
+        Yield all nginx version from its git tags.
         """
-        nginx_versions = GitHubTagsAPI().fetch("nginx/nginx")
-        for version in nginx_versions:
+        nginx_versions = package_versions.versions("pkg:github/nginx/nginx")
+        for version in nginx_versions or []:
             cleaned = clean_nginx_git_tag(version.value)
-            yield PackageVersion(value=cleaned, release_date=version.release_date)
+            yield cleaned
 
 
 class ApacheHTTPDImprover(ValidVersionImprover):

vulnerabilities/models.py

@@ -546,6 +546,9 @@ def for_cve(self, cve):
         """
         return self.filter(vulnerabilities__vulnerabilityreference__reference_id__exact=cve)
 
+    def for_purls(self, purls=[]):
+        return Package.objects.filter(package_url__in=purls).distinct()
+
 
 def get_purl_query_lookups(purl):
     """