Add SSVC trees, resource URL and max_advisories

CHANGELOG.rst

@@ -5,6 +5,8 @@ next release
 ---------------------
 
 - WARNING: Vulnerablecode V1 API and UI has stopped supporting Ubuntu OVAL advisories, please shift to V3 API for new Ubuntu advisories.
+- WARNING: We will deprecate improver pipelines for calculating package version rank, grouping advisories for packages and calculating risk scores in the next release, we are doing it at advisory import time instead of as separate pipelines, this will improve the performance and consistency of the data.
+- Calculate package verion rank, group advisories for packages and package risk score and advisory risk score during import of advisories.
 - Add attribute ``pipeline_id`` to AdvisoryV2 to track the pipeline that created the advisory, also rename existing ``datasource_id`` and AVIDs. 
 
 Version v38.6.0

vulnerabilities/api_v3.py

@@ -8,7 +8,6 @@
 #
 
 from collections import defaultdict
-from typing import List
 from urllib.parse import urlencode
 
 from django.db.models import Exists
@@ -23,21 +22,19 @@
 from rest_framework.reverse import reverse
 from rest_framework.throttling import AnonRateThrottle
 
+from vulnerabilities.models import SSVC
 from vulnerabilities.models import AdvisoryAlias
 from vulnerabilities.models import AdvisoryReference
 from vulnerabilities.models import AdvisorySet
 from vulnerabilities.models import AdvisorySetMember
 from vulnerabilities.models import AdvisorySeverity
 from vulnerabilities.models import AdvisoryV2
 from vulnerabilities.models import AdvisoryWeakness
-from vulnerabilities.models import Group
 from vulnerabilities.models import GroupedAdvisory
 from vulnerabilities.models import ImpactedPackageAffecting
 from vulnerabilities.models import PackageV2
 from vulnerabilities.throttling import PermissionBasedUserRateThrottle
 from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS
-from vulnerabilities.utils import get_advisories_from_groups
-from vulnerabilities.utils import merge_and_save_grouped_advisories
 
 
 class PackageQuerySerializer(serializers.Serializer):
@@ -48,6 +45,7 @@ class PackageQuerySerializer(serializers.Serializer):
     )
     details = serializers.BooleanField(default=False)
     ignore_qualifiers_subpath = serializers.BooleanField(default=False)
+    max_advisories = serializers.IntegerField(default=100, min_value=1, max_value=10000)
 
     def validate(self, data):
         if not data["purls"]:
@@ -227,29 +225,54 @@ def get_affected_by_vulnerabilities(self, package):
             result = []
 
             for adv in advisories:
-                fixed = impact_map.get(adv["avid"])
-                adv.pop("avid", None)
+                fixed = impact_map.get(adv["advisory_uid"]) or []
+                resource_url = None
+
+                if request := self.context.get("request", None):
+                    resource_url = adv.get("resource_url") or None
+                    resource_url = request.build_absolute_uri(location=resource_url)
 
                 result.append(
                     {
-                        **adv,
+                        "advisory_id": adv["advisory_id"],
+                        "advisory_uid": adv["advisory_uid"],
+                        "aliases": adv["aliases"],
+                        "summary": adv["summary"],
+                        "weighted_severity": adv["weighted_severity"],
+                        "exploitability": adv["exploitability"],
+                        "risk_score": adv["risk_score"],
+                        "ssvc_trees": adv["ssvc_trees"],
                         "fixed_by_packages": fixed,
+                        "resource_url": resource_url,
                     }
                 )
 
             return result
 
-        advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url)
+        if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS:
+            advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(
+                package.package_url
+            )
 
-        advisories = []
+            advisories = []
 
-        if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS:
             advisories_ids = advisories_qs.only("id")
 
             advisories_ids = list(advisories_ids[:101])
-            if len(advisories_ids) > 100:
+            if len(advisories_ids) > self.context.get("max_advisories", 100):
                 return None
 
+            advisories_qs = advisories_qs.prefetch_related(
+                "aliases",
+                Prefetch(
+                    "related_ssvcs",
+                    queryset=SSVC.objects.select_related("source_advisory")
+                    .only("id", "decision", "options", "vector", "source_advisory__url")
+                    .distinct("source_advisory__url"),
+                    to_attr="prefetched_ssvc_trees",
+                ),
+            )
+
             advisory_by_avid = {adv.avid: adv for adv in advisories_qs}
             avids = advisory_by_avid.keys()
 
@@ -265,82 +288,95 @@ def get_affected_by_vulnerabilities(self, package):
 
             for advisory in advisories_qs:
                 impact = impact_by_avid.get(advisory.avid)
-                if not impact:
-                    continue
+                fixed_by_packages = []
+                if impact:
+                    fixed_by_packages = [pkg.purl for pkg in impact.fixed_by_packages.all()]
+
+                resource_url = None
+
+                if request := self.context.get("request", None):
+                    resource_url = request.build_absolute_uri(location=advisory.get_absolute_url())
 
                 result.append(
                     {
                         "advisory_id": advisory.advisory_id.split("/")[-1],
                         "advisory_uid": advisory.avid,
                         "aliases": [alias.alias for alias in advisory.aliases.all()],
                         "summary": advisory.summary,
-                        "severity": advisory.weighted_severity,
+                        "weighted_severity": advisory.weighted_severity,
                         "exploitability": advisory.exploitability,
                         "risk_score": advisory.risk_score,
-                        "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()],
+                        "fixed_by_packages": fixed_by_packages,
+                        "resource_url": resource_url,
+                        "ssvc_trees": [
+                            {
+                                "vector": ssvc.vector,
+                                "decision": ssvc.decision,
+                                "options": ssvc.options,
+                                "source_url": ssvc.source_advisory.url,
+                            }
+                            for ssvc in advisory.prefetched_ssvc_trees
+                        ],
                     }
                 )
 
             return result
 
-        if not advisories:
-            if package.type in TYPES_WITH_MULTIPLE_IMPORTERS:
-                advisories_qs = advisories_qs.prefetch_related(
-                    "aliases",
-                    "impacted_packages__affecting_packages",
-                    "impacted_packages__fixed_by_packages",
-                )
-                advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories(
-                    package, advisories_qs, "affecting"
-                )
-                return self.return_advisories_data(package, advisories_qs, advisories)
-
     def get_fixing_vulnerabilities(self, package):
         advisories = self.context["fixing_advisory_map"].get(package.id, [])
-        if advisories:
-            return advisories
+        results = []
+        resource_url = None
+        for advisory in advisories:
+            if request := self.context.get("request", None):
+                resource_url = request.build_absolute_uri(location=advisory["resource_url"])
+            results.append(
+                {
+                    "advisory_id": advisory["advisory_id"],
+                    "resource_url": resource_url,
+                    "advisory_uid": advisory["advisory_uid"],
+                }
+            )
+        if results:
+            return results
 
         advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url)
 
-        if not package.type in TYPES_WITH_MULTIPLE_IMPORTERS:
+        if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS:
             advisories_ids = advisories_qs.only("id")
 
             advisories_ids = list(advisories_ids[:101])
-            if len(advisories_ids) > 100:
+            if len(advisories_ids) > self.context.get("max_advisories", 100):
                 return None
 
             results = []
 
             for advisory in advisories_qs:
+                resource_url = None
+                if request := self.context.get("request", None):
+                    resource_url = request.build_absolute_uri(location=advisory.get_absolute_url())
                 results.append(
                     {
                         "advisory_id": advisory.advisory_id.split("/")[-1],
                         "advisory_uid": advisory.avid,
+                        "resource_url": resource_url,
                     }
                 )
             return results
 
-        if package.type in TYPES_WITH_MULTIPLE_IMPORTERS:
-            advisories_qs = advisories_qs.prefetch_related(
-                "aliases",
-                "impacted_packages__affecting_packages",
-                "impacted_packages__fixed_by_packages",
-            )
-            if not advisories_qs.exists():
-                return []
-            advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories(
-                package, advisories_qs, "fixing"
-            )
-            return self.return_fixing_advisories_data(advisories)
-
     def return_fixing_advisories_data(self, advisories):
         result = []
         for advisory in advisories:
             assert isinstance(advisory, GroupedAdvisory)
+            resource_url = None
+            if request := self.context.get("request", None):
+                resource_url = request.build_absolute_uri(
+                    location=advisory.advisory.get_absolute_url()
+                )
             result.append(
                 {
                     "advisory_id": advisory.identifier,
                     "advisory_uid": advisory.advisory.avid,
+                    "resource_url": resource_url,
                 }
             )
 
@@ -361,22 +397,28 @@ def return_advisories_data(self, package, advisories_qs, advisories):
         result = []
         for advisory in advisories:
             assert isinstance(advisory, GroupedAdvisory)
+            resource_url = None
+            fixed_by_packages = []
+            if request := self.context.get("request", None):
+                resource_url = request.build_absolute_uri(
+                    location=advisory.advisory.get_absolute_url()
+                )
             impact = impact_by_avid.get(advisory.advisory.avid)
-            if not impact:
-                continue
+            if impact:
+                fixed_by_packages = list(set([pkg.purl for pkg in impact.fixed_by_packages.all()]))
 
             result.append(
                 {
                     "advisory_id": advisory.identifier,
                     "advisory_uid": advisory.advisory.avid,
                     "aliases": [alias.alias for alias in advisory.aliases],
+                    "summary": advisory.advisory.summary,
                     "weighted_severity": advisory.weighted_severity,
                     "exploitability": advisory.exploitability,
                     "risk_score": advisory.risk_score,
-                    "summary": advisory.advisory.summary,
-                    "fixed_by_packages": list(
-                        set([pkg.purl for pkg in impact.fixed_by_packages.all()])
-                    ),
+                    "fixed_by_packages": fixed_by_packages,
+                    "resource_url": resource_url,
+                    "ssvc_trees": advisory.ssvc_trees,
                 }
             )
 
@@ -405,13 +447,16 @@ def create(self, request, *args, **kwargs):
         purls = serializer.validated_data["purls"]
         details = serializer.validated_data["details"]
         ignore_qualifiers_subpath = serializer.validated_data["ignore_qualifiers_subpath"]
+        max_advisories = serializer.validated_data["max_advisories"]
 
         if not purls:
-            impacted = ImpactedPackageAffecting.objects.filter(package_id=OuterRef("id"))
+            latest_impacts = ImpactedPackageAffecting.objects.filter(
+                package_id=OuterRef("pk"),
+                impacted_package__is_latest=True,
+            )
 
             query = (
-                PackageV2.objects.annotate(has_vuln=Exists(impacted))
-                .filter(has_vuln=True)
+                PackageV2.objects.filter(Exists(latest_impacts))
                 .values_list("package_url", flat=True)
                 .order_by("package_url")
             )
@@ -469,6 +514,7 @@ def create(self, request, *args, **kwargs):
                 "advisory_map": affected_advisory_map,
                 "impact_map": impact_map,
                 "fixing_advisory_map": fixing_advisory_map,
+                "max_advisories": max_advisories,
             },
         )
         return self.get_paginated_response(serializer.data)
@@ -583,7 +629,27 @@ def get_affected_advisories_bulk(packages):
             relation_type="affecting",
         )
         .select_related("primary_advisory")
-        .prefetch_related(Prefetch("aliases", queryset=AdvisoryAlias.objects.only("alias")))
+        .prefetch_related(
+            Prefetch("aliases", queryset=AdvisoryAlias.objects.only("alias")),
+            Prefetch(
+                "members",
+                queryset=AdvisorySetMember.objects.select_related("advisory").prefetch_related(
+                    Prefetch(
+                        "advisory__related_ssvcs",
+                        queryset=SSVC.objects.select_related("source_advisory")
+                        .only(
+                            "id",
+                            "options",
+                            "decision",
+                            "vector",
+                            "source_advisory__url",
+                        )
+                        .distinct("source_advisory__url"),
+                        to_attr="prefetched_ssvc_trees",
+                    )
+                ),
+            ),
+        )
         .annotate(
             max_severity=Max(
                 "members__advisory__weighted_severity",
@@ -627,6 +693,22 @@ def get_affected_advisories_bulk(packages):
             identifier = primary.advisory_id.split("/")[-1]
 
             aliases = [a for a in adv._aliases_cache if a != identifier]
+            all_ssvc = []
+
+            for member in adv.members.all():
+                all_ssvc.extend(member.advisory.prefetched_ssvc_trees)
+
+            ssvcs = []
+
+            for ssvc in all_ssvc:
+                ssvcs.append(
+                    {
+                        "vector": ssvc.vector,
+                        "decision": ssvc.decision,
+                        "options": ssvc.options,
+                        "source_url": ssvc.source_advisory.url,
+                    }
+                )
 
             grouped.append(
                 {
@@ -637,6 +719,8 @@ def get_affected_advisories_bulk(packages):
                     "exploitability": exploitability,
                     "risk_score": risk_score,
                     "summary": primary.summary,
+                    "resource_url": primary.get_absolute_url(),
+                    "ssvc_trees": ssvcs,
                 }
             )
 
@@ -697,17 +781,21 @@ def get_fixing_advisories_bulk(packages):
     package_map = defaultdict(list)
 
     for adv in advisory_sets:
-        package_map[adv.package_id].append(adv.primary_advisory.advisory_id)
+        package_map[adv.package_id].append(adv.primary_advisory)
 
     result = {}
 
     for package in packages:
         groups = package_map.get(package.id, [])
         grouped = []
 
-        for adv_id in groups:
+        for advisory in groups:
             grouped.append(
-                {"advisory_id": adv_id.split("/")[-1], "advisory_uid": adv_id.split("/")[-1]}
+                {
+                    "advisory_id": advisory.advisory_id.split("/")[-1],
+                    "resource_url": advisory.get_absolute_url(),
+                    "advisory_uid": advisory.avid,
+                }
             )
 
         result[package.id] = grouped