Package purl model updates (#1368)

* Add missing migration for vulnerability.status Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Migrate qualifiers to plain charfield step 1 Create qualifiers_temp temp field Reference: #1327 Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Migrate qualifiers to plain charfield step 2 Copy qualifiers to qualifiers_temp Reference: #1327 Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Add qualifiers_temp in unique_together step 3 Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Remove qualifiers from qunique_together step 4 Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Copy qualifiers_temp to qualifiers step 5 Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Add qualifiers in unique_together step 6 Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Delete qualifiers_temp and remove it from unique_togther step 7 Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Formatting changes Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Correct the 0045 migration Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Migrate qualifiers to plain charfield step 1 Create qualifiers_temp temp field Reference: #1327 Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Migrate qualifiers to plain charfield step 2 Copy qualifiers to qualifiers_temp Reference: #1327 Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Add qualifiers_temp in unique_together step 3 Reference: #1327 Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Remove qualifiers from unique_together step 4 Reference: #1327 Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Copy qualifiers_temp to qualifiers step 5 Reference: #1327 Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Add qualifiers in unique_together step 6 Reference: #1327 Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Delete qualifiers_temp field and unique_togther step 7 Reference: #1327 Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Format models.py Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Remove dupe Packages from qualifiers Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Remove dupe Packages from ns/name Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> * Correct migrations and add tests Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Fix tests Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Update tests Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Fix tests Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Remove tests for warts Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Add changelog Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Update PR according to recent changes Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Update tests Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Address review comments Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> --------- Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com> Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> Co-authored-by: Philippe Ombredanne <pombredanne@nexb.com>
aboutcode-org · Dec 29, 2023 · 5932722 · 5932722
1 parent e2b60c9
commit 5932722
Show file tree

Hide file tree

Showing 83 changed files with 8,168 additions and 8,065 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -2,6 +2,19 @@ Release notes
 =============
 
 
+Version v34.0.0rc1
+-------------------
+
+- We updated package-url models, WARNING: in next major version of 
+  vulnerablecode i.e v35.0.0 qualifiers will be of type ``string`` and not ``dict``.
+- We fixed table borders in Vulnerability details UI #1356 (#1358)
+- We fixed import runner's process_inferences (#1360)
+- We fixed debian OVAL importer (#1361)
+- We added graph model diagrams #977(#1350)
+- We added endpoint for purl lookup (#1359)
+- We fixed swagger API docs generation (#1366)
+
+
 Version v33.6.5
 -------------------
 
@@ -12,12 +25,6 @@ Version v33.6.4
 -------------------
 
 - We added /var/www/html as volume in Docker compose (#1371).
-- We fixed table borders in Vulnerability details UI #1356 (#1358)
-- We fixed import runner's process_inferences (#1360)
-- We fixed debian OVAL importer (#1361)
-- We added graph model diagrams #977(#1350)
-- We added endpoint for purl lookup (#1359)
-- We fixed swagger API docs generation (#1366)
 
 
 Version v33.6.3

diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py
@@ -14,6 +14,7 @@
 from drf_spectacular.utils import extend_schema
 from drf_spectacular.utils import inline_serializer
 from packageurl import PackageURL
+from packageurl import normalize_qualifiers
 from rest_framework import serializers
 from rest_framework import status
 from rest_framework import viewsets
@@ -147,6 +148,11 @@ class PackageSerializer(serializers.HyperlinkedModelSerializer):
     Lookup software package using Package URLs
     """
 
+    def to_representation(self, instance):
+        data = super().to_representation(instance)
+        data["qualifiers"] = normalize_qualifiers(data["qualifiers"], encode=False)
+        return data
+
     next_non_vulnerable_version = serializers.SerializerMethodField("get_next_non_vulnerable")
 
     def get_next_non_vulnerable(self, package):
@@ -252,12 +258,13 @@ class PackageFilterSet(filters.FilterSet):
     class Meta:
         model = Package
         fields = [
-            "name",
             "type",
+            "namespace",
+            "name",
             "version",
+            "qualifiers",
             "subpath",
             "purl",
-            "namespace",
             "packagerelatedvulnerability__fix",
         ]
 

diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py
@@ -37,10 +37,11 @@
 from vulnerabilities.severity_systems import SCORING_SYSTEMS
 from vulnerabilities.severity_systems import ScoringSystem
 from vulnerabilities.utils import classproperty
-from vulnerabilities.utils import evolve_purl
 from vulnerabilities.utils import get_reference_id
 from vulnerabilities.utils import is_cve
 from vulnerabilities.utils import nearest_patched_package
+from vulnerabilities.utils import purl_to_dict
+from vulnerabilities.utils import update_purl_version
 
 logger = logging.getLogger(__name__)
 
@@ -154,8 +155,7 @@ def get_fixed_purl(self):
         """
         if not self.fixed_version:
             raise ValueError(f"Affected Package {self.package!r} does not have a fixed version")
-        fixed_purl = evolve_purl(purl=self.package, version=str(self.fixed_version))
-        return fixed_purl
+        return update_purl_version(purl=self.package, version=str(self.fixed_version))
 
     @classmethod
     def merge(
@@ -198,7 +198,7 @@ def to_dict(self):
         if self.affected_version_range:
             affected_version_range = str(self.affected_version_range)
         return {
-            "package": self.package.to_dict(),
+            "package": purl_to_dict(self.package),
             "affected_version_range": affected_version_range,
             "fixed_version": str(self.fixed_version) if self.fixed_version else None,
         }

diff --git a/vulnerabilities/improver.py b/vulnerabilities/improver.py
@@ -76,13 +76,17 @@ def to_dict(self):
         """
         Return a dict representation of this Inference
         """
+        from vulnerabilities.utils import purl_to_dict
+
         return {
             "vulnerability_id": self.vulnerability_id,
             "aliases": self.aliases,
             "confidence": self.confidence,
             "summary": self.summary,
-            "affected_purls": [affected_purl.to_dict() for affected_purl in self.affected_purls],
-            "fixed_purl": self.fixed_purl.to_dict() if self.fixed_purl else None,
+            "affected_purls": [
+                purl_to_dict(affected_purl) for affected_purl in self.affected_purls
+            ],
+            "fixed_purl": purl_to_dict(self.fixed_purl) if self.fixed_purl else None,
             "references": [ref.to_dict() for ref in self.references],
             "weaknesses": self.weaknesses,
         }

diff --git a/vulnerabilities/improvers/default.py b/vulnerabilities/improvers/default.py
@@ -23,7 +23,7 @@
 from vulnerabilities.improver import Improver
 from vulnerabilities.improver import Inference
 from vulnerabilities.models import Advisory
-from vulnerabilities.utils import evolve_purl
+from vulnerabilities.utils import update_purl_version
 
 logger = logging.getLogger(__name__)
 
@@ -120,14 +120,16 @@ def get_exact_purls(affected_package: AffectedPackage) -> Tuple[List[PackageURL]
             fixed_versions = [c.version for c in vr.constraints if c and c.comparator == "!="]
             resolved_versions = [v for v in range_versions if v and v in vr]
             for version in resolved_versions:
-                affected_purl = evolve_purl(purl=affected_package.package, version=str(version))
+                affected_purl = update_purl_version(
+                    purl=affected_package.package, version=str(version)
+                )
                 affected_purls.append(affected_purl)
 
         if affected_package.fixed_version:
             fixed_versions.append(affected_package.fixed_version)
 
         fixed_purls = [
-            evolve_purl(purl=affected_package.package, version=str(version))
+            update_purl_version(purl=affected_package.package, version=str(version))
             for version in fixed_versions
         ]
         return affected_purls, fixed_purls

diff --git a/vulnerabilities/improvers/valid_versions.py b/vulnerabilities/improvers/valid_versions.py
@@ -44,11 +44,11 @@
 from vulnerabilities.models import Advisory
 from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage
 from vulnerabilities.utils import clean_nginx_git_tag
-from vulnerabilities.utils import evolve_purl
 from vulnerabilities.utils import get_affected_packages_by_patched_package
 from vulnerabilities.utils import is_vulnerable_nginx_version
 from vulnerabilities.utils import nearest_patched_package
 from vulnerabilities.utils import resolve_version_range
+from vulnerabilities.utils import update_purl_version
 
 logger = logging.getLogger(__name__)
 
@@ -253,13 +253,13 @@ def get_inferences_from_versions(
                     affected_version_range=affected_version_range,
                     fixed_versions=fixed_versions,
                 ):
-                    new_purl = evolve_purl(purl=purl, version=str(version))
+                    new_purl = update_purl_version(purl=purl, version=str(version))
                     affected_purls.append(new_purl)
 
         # TODO: This also yields with a lower fixed version, maybe we should
         # only yield fixes that are upgrades ?
         for fixed_version in fixed_versions:
-            fixed_purl = evolve_purl(purl=purl, version=str(fixed_version))
+            fixed_purl = update_purl_version(purl=purl, version=str(fixed_version))
 
             yield Inference.from_advisory_data(
                 advisory_data,

diff --git a/vulnerabilities/migrations/0022_alter_vulnerability_vulnerability_id.py b/vulnerabilities/migrations/0022_alter_vulnerability_vulnerability_id.py
@@ -14,6 +14,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='vulnerability',
             name='vulnerability_id',
-            field=models.CharField(blank=True, default=vulnerabilities.models.build_vcid, help_text='Unique identifier for a vulnerability in the external representation. It is prefixed with VCID-', max_length=20, unique=True),
+            field=models.CharField(blank=True, default=vulnerabilities.models.utils.build_vcid, help_text='Unique identifier for a vulnerability in the external representation. It is prefixed with VCID-', max_length=20, unique=True),
         ),
     ]
diff --git a/vulnerabilities/migrations/0045_remove_duplicated_purls_with_same_qualifiers.py b/vulnerabilities/migrations/0045_remove_duplicated_purls_with_same_qualifiers.py
@@ -0,0 +1,48 @@
+# Generated by Django 4.1.13 on 2023-12-05 14:54
+
+from itertools import groupby
+
+from django.db import migrations
+from django.db.models import Count
+
+
+class Migration(migrations.Migration):
+
+    def remove_dupes(apps, schema_editor):
+        """
+        Remove duplicated Package with the same purl and the same "serialized"
+        qualifiers. Some qualifiers have been stored in the JSON field as
+        dicts/objects and some have been seralized as query strings/normalized
+        qualifiers and then just stored as a JSON string.
+
+        We are keeping the JSON dict over the string variant.
+        """
+        Package = apps.get_model("vulnerabilities", "Package")
+
+        duplicates = (
+            Package.objects
+            .exclude(qualifiers__in=("", None, {}))
+            .values_list("package_url")
+            .order_by("package_url")
+            .annotate(count_id=Count("id"))
+            .filter(count_id__gt=1)
+        )
+        to_delete = []
+        # Get all rows with the same purl,
+        # delete the qualifier(s) that are a string
+        # and keep the others
+        for purl, _cid in duplicates:
+            for package in Package.objects.filter(package_url=purl):
+                if isinstance(package.qualifiers, str):
+                    to_delete.append(package.id)
+
+        deleted, _ = Package.objects.filter(id__in=to_delete).delete()
+        print(f"Deleted {deleted} duplicated Packages")
+
+    dependencies = [
+        ("vulnerabilities", "0044_alter_packagechangelog_options_and_more"),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_dupes, reverse_code=migrations.RunPython.noop),
+    ]
diff --git a/vulnerabilities/migrations/0046_package_qualifiers_temp.py b/vulnerabilities/migrations/0046_package_qualifiers_temp.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.1.13 on 2023-12-05 11:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0045_remove_duplicated_purls_with_same_qualifiers"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="package",
+            name="qualifiers_temp",
+            field=models.CharField(blank=True, max_length=1024),
+        ),
+    ]
diff --git a/vulnerabilities/migrations/0047_copy_qualifiers_to_qualifiers_temp.py b/vulnerabilities/migrations/0047_copy_qualifiers_to_qualifiers_temp.py
@@ -0,0 +1,36 @@
+# Generated by Django 4.1.13 on 2023-12-05 11:42
+
+from django.db import migrations
+from packageurl import normalize_qualifiers
+
+
+class Migration(migrations.Migration):
+
+    def copy_qualifiers(apps, schema_editor):
+        """
+        Bulk update qualifiers_temp from the legacy JSON field
+        """
+        Package = apps.get_model("vulnerabilities", "Package")
+        updatables = []
+        for package in Package.objects.all():
+            qualifiers = package.qualifiers
+            normalized_string = normalize_qualifiers(qualifiers, encode=True) or ""
+            package.qualifiers_temp = normalized_string
+            updatables.append(package)
+
+        updated = Package.objects.bulk_update(
+            objs = updatables,
+            fields=["qualifiers_temp",], 
+            batch_size=500,
+        )
+        print(f"Copied {updated} qualifiers to qualifiers_temp")            
+
+
+
+    dependencies = [
+        ("vulnerabilities", "0046_package_qualifiers_temp"),
+    ]
+
+    operations = [
+        migrations.RunPython(copy_qualifiers, reverse_code=migrations.RunPython.noop),
+    ]
diff --git a/vulnerabilities/migrations/0048_alter_package_unique_together.py b/vulnerabilities/migrations/0048_alter_package_unique_together.py
@@ -0,0 +1,19 @@
+# Generated by Django 4.1.7 on 2023-12-05 13:42
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0047_copy_qualifiers_to_qualifiers_temp"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="package",
+            unique_together={
+                ("type", "namespace", "name", "version", "qualifiers", "subpath", "qualifiers_temp")
+            },
+        ),
+    ]
diff --git a/vulnerabilities/migrations/0049_alter_package_unique_together_and_more.py b/vulnerabilities/migrations/0049_alter_package_unique_together_and_more.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.1.7 on 2023-12-05 13:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0048_alter_package_unique_together"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="package",
+            unique_together={
+                ("type", "namespace", "name", "version", "subpath", "qualifiers_temp")
+            },
+        ),
+        migrations.AlterField(
+            model_name="package",
+            name="qualifiers",
+            field=models.CharField(
+                blank=True,
+                help_text="Extra qualifying data for a package such as the name of an OS, architecture, distro, etc.",
+                max_length=1024,
+            ),
+        ),
+    ]
diff --git a/vulnerabilities/migrations/0050_copy_qualifiers_temp_back_to_qualifiers.py b/vulnerabilities/migrations/0050_copy_qualifiers_temp_back_to_qualifiers.py
@@ -0,0 +1,35 @@
+# Generated by Django 4.1.13 on 2023-12-05 11:42
+
+from django.db import migrations
+from packageurl import normalize_qualifiers
+
+
+class Migration(migrations.Migration):
+
+    def copy_qualifiers_temp(apps, schema_editor):
+        """
+        Bulk update qualifiers_temp from the legacy JSON field
+        """
+        Package = apps.get_model("vulnerabilities", "Package")
+        updatables = []
+        for package in Package.objects.all():
+            qualifiers_temp = package.qualifiers_temp
+            package.qualifiers = qualifiers_temp
+            updatables.append(package)
+
+        updated = Package.objects.bulk_update(
+            objs = updatables,
+            fields=["qualifiers",], 
+            batch_size=500,
+        )
+        print(f"Copied {updated} qualifiers_temp to qualifiers")            
+
+
+
+    dependencies = [
+        ("vulnerabilities", "0049_alter_package_unique_together_and_more"),
+    ]
+
+    operations = [
+        migrations.RunPython(copy_qualifiers_temp, reverse_code=migrations.RunPython.noop),
+    ]
diff --git a/vulnerabilities/migrations/0051_alter_package_unique_together_and_more.py b/vulnerabilities/migrations/0051_alter_package_unique_together_and_more.py
@@ -0,0 +1,21 @@
+# Generated by Django 4.1.13 on 2023-12-19 09:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0050_copy_qualifiers_temp_back_to_qualifiers"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="package",
+            unique_together={("type", "namespace", "name", "version", "qualifiers", "subpath")},
+        ),
+        migrations.RemoveField(
+            model_name="package",
+            name="qualifiers_temp",
+        ),
+    ]