From 08df01dc9e7a8604176491c7f460a262f1e815cb Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 29 Apr 2026 17:21:16 +0530 Subject: [PATCH 01/24] Add SSVC trees, resource URL and max_advisories Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 132 +++++++++++++++++++++++++++++++++----- vulnerabilities/models.py | 3 +- vulnerabilities/utils.py | 35 +++++++++- 3 files changed, 151 insertions(+), 19 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 634476ddf..420b8bcab 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -23,6 +23,7 @@ from rest_framework.reverse import reverse from rest_framework.throttling import AnonRateThrottle +from vulnerabilities.models import SSVC from vulnerabilities.models import AdvisoryAlias from vulnerabilities.models import AdvisoryReference from vulnerabilities.models import AdvisorySet @@ -30,13 +31,11 @@ from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import AdvisoryWeakness -from vulnerabilities.models import Group from vulnerabilities.models import GroupedAdvisory from vulnerabilities.models import ImpactedPackageAffecting from vulnerabilities.models import PackageV2 from vulnerabilities.throttling import PermissionBasedUserRateThrottle from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS -from vulnerabilities.utils import get_advisories_from_groups from vulnerabilities.utils import merge_and_save_grouped_advisories @@ -48,6 +47,7 @@ class PackageQuerySerializer(serializers.Serializer): ) details = serializers.BooleanField(default=False) ignore_qualifiers_subpath = serializers.BooleanField(default=False) + max_advisories = serializers.IntegerField(default=100, min_value=1, max_value=10000) def validate(self, data): if not data["purls"]: @@ -229,11 +229,17 @@ def get_affected_by_vulnerabilities(self, package): for adv in advisories: fixed = impact_map.get(adv["avid"]) adv.pop("avid", None) + resource_url = None + + if request := self.context.get("request", None): + resource_url = adv.pop("resource_url", None) + resource_url = request.build_absolute_uri(location=resource_url) result.append( { **adv, "fixed_by_packages": fixed, + "resource_url": resource_url, } ) @@ -247,9 +253,20 @@ def get_affected_by_vulnerabilities(self, package): advisories_ids = advisories_qs.only("id") advisories_ids = list(advisories_ids[:101]) - if len(advisories_ids) > 100: + if len(advisories_ids) > self.context.get("max_advisories", 100): return None + advisories_qs = advisories_qs.prefetch_related( + "aliases", + Prefetch( + "related_ssvcs", + queryset=SSVC.objects.select_related("source_advisory").only( + "id", "decision", "options", "vector", "source_advisory__url" + ), + to_attr="prefetched_ssvc_trees", + ), + ) + advisory_by_avid = {adv.avid: adv for adv in advisories_qs} avids = advisory_by_avid.keys() @@ -265,8 +282,14 @@ def get_affected_by_vulnerabilities(self, package): for advisory in advisories_qs: impact = impact_by_avid.get(advisory.avid) - if not impact: - continue + fixed_by_packages = [] + if impact: + fixed_by_packages = [pkg.purl for pkg in impact.fixed_by_packages.all()] + + resource_url = None + + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri(location=advisory.get_absolute_url()) result.append( { @@ -277,7 +300,17 @@ def get_affected_by_vulnerabilities(self, package): "severity": advisory.weighted_severity, "exploitability": advisory.exploitability, "risk_score": advisory.risk_score, - "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], + "fixed_by_packages": fixed_by_packages, + "resource_url": resource_url, + "ssvc_trees": [ + { + "vector": ssvc.vector, + "decision": ssvc.decision, + "options": ssvc.options, + "source_url": ssvc.source_advisory.url, + } + for ssvc in advisory.prefetched_ssvc_trees + ], } ) @@ -297,8 +330,18 @@ def get_affected_by_vulnerabilities(self, package): def get_fixing_vulnerabilities(self, package): advisories = self.context["fixing_advisory_map"].get(package.id, []) - if advisories: - return advisories + results = [] + for advisory in advisories: + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri(location=advisory["resource_url"]) + results.append( + { + "advisory_id": advisory["advisory_id"], + "resource_url": resource_url, + } + ) + if results: + return results advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) @@ -306,16 +349,19 @@ def get_fixing_vulnerabilities(self, package): advisories_ids = advisories_qs.only("id") advisories_ids = list(advisories_ids[:101]) - if len(advisories_ids) > 100: + if len(advisories_ids) > self.context.get("max_advisories", 100): return None results = [] for advisory in advisories_qs: + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri(location=advisory.get_absolute_url()) results.append( { "advisory_id": advisory.advisory_id.split("/")[-1], "advisory_uid": advisory.avid, + "resource_url": resource_url, } ) return results @@ -337,10 +383,16 @@ def return_fixing_advisories_data(self, advisories): result = [] for advisory in advisories: assert isinstance(advisory, GroupedAdvisory) + resource_url = None + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri( + location=advisory.advisory.get_absolute_url() + ) result.append( { "advisory_id": advisory.identifier, "advisory_uid": advisory.advisory.avid, + "resource_url": resource_url, } ) @@ -361,9 +413,15 @@ def return_advisories_data(self, package, advisories_qs, advisories): result = [] for advisory in advisories: assert isinstance(advisory, GroupedAdvisory) + resource_url = None + fixed_by_packages = [] + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri( + location=advisory.advisory.get_absolute_url() + ) impact = impact_by_avid.get(advisory.advisory.avid) if not impact: - continue + fixed_by_packages = list(set([pkg.purl for pkg in impact.fixed_by_packages.all()])) result.append( { @@ -374,9 +432,9 @@ def return_advisories_data(self, package, advisories_qs, advisories): "exploitability": advisory.exploitability, "risk_score": advisory.risk_score, "summary": advisory.advisory.summary, - "fixed_by_packages": list( - set([pkg.purl for pkg in impact.fixed_by_packages.all()]) - ), + "fixed_by_packages": fixed_by_packages, + "resource_url": resource_url, + "ssvc_trees": advisory.ssvc_trees, } ) @@ -405,6 +463,7 @@ def create(self, request, *args, **kwargs): purls = serializer.validated_data["purls"] details = serializer.validated_data["details"] ignore_qualifiers_subpath = serializer.validated_data["ignore_qualifiers_subpath"] + max_advisories = serializer.validated_data["max_advisories"] if not purls: impacted = ImpactedPackageAffecting.objects.filter(package_id=OuterRef("id")) @@ -469,6 +528,7 @@ def create(self, request, *args, **kwargs): "advisory_map": affected_advisory_map, "impact_map": impact_map, "fixing_advisory_map": fixing_advisory_map, + "max_advisories": max_advisories, }, ) return self.get_paginated_response(serializer.data) @@ -583,7 +643,25 @@ def get_affected_advisories_bulk(packages): relation_type="affecting", ) .select_related("primary_advisory") - .prefetch_related(Prefetch("aliases", queryset=AdvisoryAlias.objects.only("alias"))) + .prefetch_related( + Prefetch("aliases", queryset=AdvisoryAlias.objects.only("alias")), + Prefetch( + "members", + queryset=AdvisorySetMember.objects.select_related("advisory").prefetch_related( + Prefetch( + "advisory__related_ssvcs", + queryset=SSVC.objects.select_related("source_advisory").only( + "id", + "options", + "decision", + "vector", + "source_advisory__url", + ), + to_attr="prefetched_ssvc_trees", + ) + ), + ), + ) .annotate( max_severity=Max( "members__advisory__weighted_severity", @@ -627,6 +705,20 @@ def get_affected_advisories_bulk(packages): identifier = primary.advisory_id.split("/")[-1] aliases = [a for a in adv._aliases_cache if a != identifier] + all_ssvc = [] + + for member in adv.members.all(): + all_ssvc.extend(member.advisory.prefetched_ssvc_trees) + + for ssvc in all_ssvc: + all_ssvc.append( + { + "vector": ssvc.vector, + "decision": ssvc.decision, + "options": ssvc.options, + "source_url": ssvc.source_advisory.url, + } + ) grouped.append( { @@ -637,6 +729,8 @@ def get_affected_advisories_bulk(packages): "exploitability": exploitability, "risk_score": risk_score, "summary": primary.summary, + "resource_url": primary.get_absolute_url(), + "ssvc_trees": all_ssvc, } ) @@ -697,7 +791,7 @@ def get_fixing_advisories_bulk(packages): package_map = defaultdict(list) for adv in advisory_sets: - package_map[adv.package_id].append(adv.primary_advisory.advisory_id) + package_map[adv.package_id].append(adv.primary_advisory) result = {} @@ -705,9 +799,13 @@ def get_fixing_advisories_bulk(packages): groups = package_map.get(package.id, []) grouped = [] - for adv_id in groups: + for advisory in groups: grouped.append( - {"advisory_id": adv_id.split("/")[-1], "advisory_uid": adv_id.split("/")[-1]} + { + "advisory_id": advisory.advisory_id.split("/")[-1], + "resource_url": advisory.get_absolute_url(), + "advisory_uid": advisory.avid, + } ) result[package.id] = grouped diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 69253f54b..8cfdf2335 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -19,7 +19,7 @@ from itertools import groupby from operator import attrgetter from traceback import format_exc as traceback_format_exc -from typing import List +from typing import Dict, List from typing import NamedTuple from typing import Optional from typing import Set @@ -3872,6 +3872,7 @@ class GroupedAdvisory(NamedTuple): weighted_severity: Optional[float] exploitability: Optional[float] risk_score: Optional[float] + ssvc_trees: List[Dict] class AdvisoryPOC(models.Model): diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 2e618a920..48d3b5384 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -35,6 +35,7 @@ import urllib3 from cwe2.database import Database from cwe2.database import InvalidCWEError +from django.db.models import Prefetch from packageurl import PackageURL from packageurl.contrib.django.utils import without_empty_values from univers.version_range import RANGE_CLASS_BY_SCHEMES @@ -979,10 +980,12 @@ def get_merged_identifier_groups(advisories): return final_groups -def get_advisories_from_groups(groups): +def get_advisories_from_groups(groups, include_ssvc_trees=False): """ Return a list of advisories from the merged groups of advisories. """ + from vulnerabilities.models import SSVC + from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import Group from vulnerabilities.models import GroupedAdvisory @@ -1016,6 +1019,35 @@ def get_advisories_from_groups(groups): identifier = group.primary.advisory_id.split("/")[-1] filtered_aliases = [alias for alias in group.aliases if alias.alias != identifier] + ssvc_trees = [] + + if include_ssvc_trees: + + all_advs = [group.primary] + list(group.secondaries) + + advisories_qs = AdvisoryV2.objects.filter( + id__in=[adv.id for adv in all_advs] + ).prefetch_related( + Prefetch( + "related_ssvcs", + queryset=SSVC.objects.select_related("source_advisory") + .only("id", "vector", "decision", "options", "source_advisory__url") + .distinct(), + to_attr="ssvc_trees", + ) + ) + + ssvc_trees = [ + { + "vector": ssvc.vector, + "decision": ssvc.decision, + "options": ssvc.options, + "url": ssvc.source_advisory.url if ssvc.source_advisory else None, + } + for adv in advisories_qs + for ssvc in adv.ssvc_trees + ] + advisories.append( GroupedAdvisory( aliases=filtered_aliases, @@ -1024,6 +1056,7 @@ def get_advisories_from_groups(groups): weighted_severity=weighted_severity, exploitability=exploitability, risk_score=risk_score, + ssvc_trees=ssvc_trees or [], ) ) From bee349145057f5dc7f6d9e153b1f154b4eb04842 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 29 Apr 2026 17:29:23 +0530 Subject: [PATCH 02/24] Fix formatting issues Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 8cfdf2335..b3ec2b525 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -19,7 +19,8 @@ from itertools import groupby from operator import attrgetter from traceback import format_exc as traceback_format_exc -from typing import Dict, List +from typing import Dict +from typing import List from typing import NamedTuple from typing import Optional from typing import Set From 557ab9e68bbba6bacc52b225a91a1bb2ef57224c Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 29 Apr 2026 18:23:23 +0530 Subject: [PATCH 03/24] Fix SSVC trees issue Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 420b8bcab..3cc093bd9 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -710,8 +710,10 @@ def get_affected_advisories_bulk(packages): for member in adv.members.all(): all_ssvc.extend(member.advisory.prefetched_ssvc_trees) + ssvcs = [] + for ssvc in all_ssvc: - all_ssvc.append( + ssvcs.append( { "vector": ssvc.vector, "decision": ssvc.decision, @@ -730,7 +732,7 @@ def get_affected_advisories_bulk(packages): "risk_score": risk_score, "summary": primary.summary, "resource_url": primary.get_absolute_url(), - "ssvc_trees": all_ssvc, + "ssvc_trees": ssvcs, } ) From c9a7003960287d9ecff9f8b29e479602794b7c8b Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 30 Apr 2026 21:51:49 +0530 Subject: [PATCH 04/24] Add unique SSVC trees only Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 3cc093bd9..11385c17c 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -262,7 +262,7 @@ def get_affected_by_vulnerabilities(self, package): "related_ssvcs", queryset=SSVC.objects.select_related("source_advisory").only( "id", "decision", "options", "vector", "source_advisory__url" - ), + ).distinct("source_advisory__url"), to_attr="prefetched_ssvc_trees", ), ) @@ -656,7 +656,7 @@ def get_affected_advisories_bulk(packages): "decision", "vector", "source_advisory__url", - ), + ).distinct("source_advisory__url"), to_attr="prefetched_ssvc_trees", ) ), From 3a9496a85f77cf2668d7e5996695780bbe04c6ff Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 1 May 2026 12:58:18 +0530 Subject: [PATCH 05/24] Fix formatting issues Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 11385c17c..7736f6408 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -260,9 +260,9 @@ def get_affected_by_vulnerabilities(self, package): "aliases", Prefetch( "related_ssvcs", - queryset=SSVC.objects.select_related("source_advisory").only( - "id", "decision", "options", "vector", "source_advisory__url" - ).distinct("source_advisory__url"), + queryset=SSVC.objects.select_related("source_advisory") + .only("id", "decision", "options", "vector", "source_advisory__url") + .distinct("source_advisory__url"), to_attr="prefetched_ssvc_trees", ), ) @@ -650,13 +650,15 @@ def get_affected_advisories_bulk(packages): queryset=AdvisorySetMember.objects.select_related("advisory").prefetch_related( Prefetch( "advisory__related_ssvcs", - queryset=SSVC.objects.select_related("source_advisory").only( + queryset=SSVC.objects.select_related("source_advisory") + .only( "id", "options", "decision", "vector", "source_advisory__url", - ).distinct("source_advisory__url"), + ) + .distinct("source_advisory__url"), to_attr="prefetched_ssvc_trees", ) ), From 5e98ecd978dc862730c8d35f05724f63b51b214d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 1 May 2026 19:22:38 +0530 Subject: [PATCH 06/24] Add avid in api Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 7736f6408..1572eb582 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -227,8 +227,7 @@ def get_affected_by_vulnerabilities(self, package): result = [] for adv in advisories: - fixed = impact_map.get(adv["avid"]) - adv.pop("avid", None) + fixed = impact_map.get(adv["avid"]) or [] resource_url = None if request := self.context.get("request", None): @@ -293,6 +292,7 @@ def get_affected_by_vulnerabilities(self, package): result.append( { + "avid": advisory.avid, "advisory_id": advisory.advisory_id.split("/")[-1], "advisory_uid": advisory.avid, "aliases": [alias.alias for alias in advisory.aliases.all()], @@ -338,6 +338,7 @@ def get_fixing_vulnerabilities(self, package): { "advisory_id": advisory["advisory_id"], "resource_url": resource_url, + "avid": advisory["avid"], } ) if results: @@ -362,6 +363,7 @@ def get_fixing_vulnerabilities(self, package): "advisory_id": advisory.advisory_id.split("/")[-1], "advisory_uid": advisory.avid, "resource_url": resource_url, + "avid": advisory.avid, } ) return results @@ -393,6 +395,7 @@ def return_fixing_advisories_data(self, advisories): "advisory_id": advisory.identifier, "advisory_uid": advisory.advisory.avid, "resource_url": resource_url, + "avid": advisory.advisory.avid, } ) @@ -420,7 +423,7 @@ def return_advisories_data(self, package, advisories_qs, advisories): location=advisory.advisory.get_absolute_url() ) impact = impact_by_avid.get(advisory.advisory.avid) - if not impact: + if impact: fixed_by_packages = list(set([pkg.purl for pkg in impact.fixed_by_packages.all()])) result.append( From d759ea498cab39f58e32917d3a1b493b38db5a71 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 26 May 2026 14:17:01 +0530 Subject: [PATCH 07/24] Fix avid errors Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 10 ++-- ...isory_id_alter_advisoryv2_avid_and_more.py | 48 +++++++++++++++++++ vulnerabilities/models.py | 8 ++-- 3 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 vulnerabilities/migrations/0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more.py diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 1572eb582..29bfb2746 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -227,7 +227,7 @@ def get_affected_by_vulnerabilities(self, package): result = [] for adv in advisories: - fixed = impact_map.get(adv["avid"]) or [] + fixed = impact_map.get(adv["advisory_uid"]) or [] resource_url = None if request := self.context.get("request", None): @@ -292,7 +292,6 @@ def get_affected_by_vulnerabilities(self, package): result.append( { - "avid": advisory.avid, "advisory_id": advisory.advisory_id.split("/")[-1], "advisory_uid": advisory.avid, "aliases": [alias.alias for alias in advisory.aliases.all()], @@ -338,7 +337,7 @@ def get_fixing_vulnerabilities(self, package): { "advisory_id": advisory["advisory_id"], "resource_url": resource_url, - "avid": advisory["avid"], + "advisory_uid": advisory["advisory_uid"], } ) if results: @@ -346,7 +345,7 @@ def get_fixing_vulnerabilities(self, package): advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) - if not package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: advisories_ids = advisories_qs.only("id") advisories_ids = list(advisories_ids[:101]) @@ -356,6 +355,7 @@ def get_fixing_vulnerabilities(self, package): results = [] for advisory in advisories_qs: + resource_url = None if request := self.context.get("request", None): resource_url = request.build_absolute_uri(location=advisory.get_absolute_url()) results.append( @@ -363,7 +363,6 @@ def get_fixing_vulnerabilities(self, package): "advisory_id": advisory.advisory_id.split("/")[-1], "advisory_uid": advisory.avid, "resource_url": resource_url, - "avid": advisory.avid, } ) return results @@ -395,7 +394,6 @@ def return_fixing_advisories_data(self, advisories): "advisory_id": advisory.identifier, "advisory_uid": advisory.advisory.avid, "resource_url": resource_url, - "avid": advisory.advisory.avid, } ) diff --git a/vulnerabilities/migrations/0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more.py b/vulnerabilities/migrations/0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more.py new file mode 100644 index 000000000..8f45487b2 --- /dev/null +++ b/vulnerabilities/migrations/0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 5.2.11 on 2026-05-26 08:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0132_migrate_advisoryv2_datasource_ids"), + ] + + operations = [ + migrations.AlterField( + model_name="advisoryv2", + name="advisory_id", + field=models.CharField( + db_index=True, + help_text="An advisory is a unique vulnerability identifier in some database, such as PYSEC-2020-2233", + max_length=200, + ), + ), + migrations.AlterField( + model_name="advisoryv2", + name="avid", + field=models.CharField( + help_text="Unique ID for the datasource used for this advisory .e.g.: pysec_importer_v2/PYSEC-2020-2233", + max_length=250, + ), + ), + migrations.AlterField( + model_name="advisoryv2", + name="datasource_id", + field=models.CharField( + db_index=True, + help_text="Unique ID for the datasource used for this advisory .e.g.: nginx", + max_length=50, + ), + ), + migrations.AlterField( + model_name="advisoryv2", + name="pipeline_id", + field=models.CharField( + db_index=True, + help_text="Unique ID for the pipeline used for this advisory .e.g.: nginx_importer_v2", + max_length=50, + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index b3ec2b525..8ed6d2520 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -3055,7 +3055,7 @@ class AdvisoryV2(models.Model): # This is similar to a type or a namespace datasource_id = models.CharField( - max_length=200, + max_length=50, blank=False, null=False, db_index=True, @@ -3063,7 +3063,7 @@ class AdvisoryV2(models.Model): ) pipeline_id = models.CharField( - max_length=200, + max_length=50, blank=False, null=False, db_index=True, @@ -3072,7 +3072,7 @@ class AdvisoryV2(models.Model): # This is similar to a name advisory_id = models.CharField( - max_length=500, + max_length=200, blank=False, null=False, unique=False, @@ -3082,7 +3082,7 @@ class AdvisoryV2(models.Model): ) avid = models.CharField( - max_length=500, + max_length=250, blank=False, null=False, help_text="Unique ID for the datasource used for this advisory ." From eb4dcc43fbcd1dc520d3937a3ecb7553bd14e53d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 27 May 2026 15:57:57 +0530 Subject: [PATCH 08/24] Group advisories at time of advisory insertion Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 62 +++++++++------- .../group_advisories_for_packages.py | 33 +-------- .../v2_improvers/unfurl_version_range.py | 6 ++ vulnerabilities/pipes/advisory.py | 5 ++ vulnerabilities/pipes/group_advisories.py | 59 +++++++++++++++- vulnerabilities/utils.py | 24 ------- vulnerabilities/views.py | 70 +++++++++---------- 7 files changed, 139 insertions(+), 120 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 29bfb2746..beb65f91e 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -231,12 +231,19 @@ def get_affected_by_vulnerabilities(self, package): resource_url = None if request := self.context.get("request", None): - resource_url = adv.pop("resource_url", None) + resource_url = adv.get("resource_url") or None resource_url = request.build_absolute_uri(location=resource_url) result.append( { - **adv, + "advisory_id": adv["identifier"], + "advisory_uid": adv["advisory_uid"], + "aliases": adv["aliases"], + "summary": adv["summary"], + "weighted_severity": adv["weighted_severity"], + "exploitability": adv["exploitability"], + "risk_score": adv["risk_score"], + "ssvc_trees": adv["ssvc_trees"], "fixed_by_packages": fixed, "resource_url": resource_url, } @@ -296,7 +303,7 @@ def get_affected_by_vulnerabilities(self, package): "advisory_uid": advisory.avid, "aliases": [alias.alias for alias in advisory.aliases.all()], "summary": advisory.summary, - "severity": advisory.weighted_severity, + "weighted_severity": advisory.weighted_severity, "exploitability": advisory.exploitability, "risk_score": advisory.risk_score, "fixed_by_packages": fixed_by_packages, @@ -315,21 +322,22 @@ def get_affected_by_vulnerabilities(self, package): return result - if not advisories: - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - advisories_qs = advisories_qs.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, advisories_qs, "affecting" - ) - return self.return_advisories_data(package, advisories_qs, advisories) + # if not advisories: + # if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + # advisories_qs = advisories_qs.prefetch_related( + # "aliases", + # "impacted_packages__affecting_packages", + # "impacted_packages__fixed_by_packages", + # ) + # advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( + # package, advisories_qs, "affecting" + # ) + # return self.return_advisories_data(package, advisories_qs, advisories) def get_fixing_vulnerabilities(self, package): advisories = self.context["fixing_advisory_map"].get(package.id, []) results = [] + resource_url = None for advisory in advisories: if request := self.context.get("request", None): resource_url = request.build_absolute_uri(location=advisory["resource_url"]) @@ -367,18 +375,18 @@ def get_fixing_vulnerabilities(self, package): ) return results - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - advisories_qs = advisories_qs.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - if not advisories_qs.exists(): - return [] - advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, advisories_qs, "fixing" - ) - return self.return_fixing_advisories_data(advisories) + # if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + # advisories_qs = advisories_qs.prefetch_related( + # "aliases", + # "impacted_packages__affecting_packages", + # "impacted_packages__fixed_by_packages", + # ) + # if not advisories_qs.exists(): + # return [] + # advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( + # package, advisories_qs, "fixing" + # ) + # return self.return_fixing_advisories_data(advisories) def return_fixing_advisories_data(self, advisories): result = [] @@ -429,10 +437,10 @@ def return_advisories_data(self, package, advisories_qs, advisories): "advisory_id": advisory.identifier, "advisory_uid": advisory.advisory.avid, "aliases": [alias.alias for alias in advisory.aliases], + "summary": advisory.advisory.summary, "weighted_severity": advisory.weighted_severity, "exploitability": advisory.exploitability, "risk_score": advisory.risk_score, - "summary": advisory.advisory.summary, "fixed_by_packages": fixed_by_packages, "resource_url": resource_url, "ssvc_trees": advisory.ssvc_trees, diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index ea6fc9185..b85c2d538 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -7,15 +7,10 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from typing import List - -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import Group from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set +from vulnerabilities.pipes.group_advisories import group_advisory_for_package from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS -from vulnerabilities.utils import merge_advisories class GroupAdvisoriesForPackages(VulnerableCodePipeline): @@ -33,28 +28,4 @@ def group_advisories_for_packages(self): def group_advisoris_for_packages(logger=None): for package in PackageV2.objects.filter(type__in=TYPES_WITH_MULTIPLE_IMPORTERS).iterator(): - logger(f"Grouping advisories for package {package.purl}") - affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( - purl=package.purl - ).prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( - purl=package.purl - ).prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - try: - affected_groups: List[Group] = merge_advisories(affecting_advisories, package) - fixed_by_groups: List[Group] = merge_advisories(fixed_by_advisories, package) - delete_and_save_advisory_set(affected_groups, package, relation="affecting") - delete_and_save_advisory_set(fixed_by_groups, package, relation="fixing") - except Exception as e: - logger(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") - continue + group_advisory_for_package(package, logger=logger) diff --git a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py index 9d874c635..3540b45b7 100644 --- a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py +++ b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py @@ -14,6 +14,7 @@ from aboutcode.pipeline import LoopProgress from django.db.models import F from django.db.models import Q +from django.db import transaction from django.utils import timezone from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS as FETCHCODE_SUPPORTED_ECOSYSTEMS from packageurl import PackageURL @@ -26,6 +27,7 @@ from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipes.fetchcode_utils import get_versions +from vulnerabilities.pipes.group_advisories import group_advisory_for_package from vulnerabilities.utils import update_purl_version @@ -159,6 +161,7 @@ def get_purl_versions(purl, cached_versions, logger): return cached_versions.get(purl) or [] +@transaction.atomic def bulk_create_with_m2m(purls, impact, relation, logger): """Bulk create PackageV2 and also bulk populate M2M Impact and Package relationships.""" if not purls: @@ -175,6 +178,9 @@ def bulk_create_with_m2m(purls, impact, relation, logger): relation(impacted_package=impact, package=package) for package in affected_packages_v2 ] + for pkg in affected_packages_v2: + group_advisory_for_package(pkg, logger=logger) + try: relation.objects.bulk_create(relations, ignore_conflicts=True) except Exception as e: diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 732e2e0ab..d4f45a38c 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -47,6 +47,7 @@ from vulnerabilities.models import VulnerabilityRelatedReference from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness +from vulnerabilities.pipes.group_advisories import group_advisory_for_package from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 @@ -387,6 +388,10 @@ def insert_advisory_v2( purls=package_fixed_purls ) + for pkg in list(affected_packages_v2) + list(fixed_packages_v2): + logger(f"Grouping advisories for package: {pkg.purl}") + group_advisory_for_package(pkg, logger=logger) + impact.affecting_packages.add(*affected_packages_v2) impact.fixed_by_packages.add(*fixed_packages_v2) diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py index 983ac3386..d72f0872b 100644 --- a/vulnerabilities/pipes/group_advisories.py +++ b/vulnerabilities/pipes/group_advisories.py @@ -7,14 +7,16 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from collections import defaultdict +from typing import List + from django.db import transaction +from vulnerabilities.models import AdvisorySet, AdvisoryV2, AdvisorySetMember +from vulnerabilities.models import Group @transaction.atomic def delete_and_save_advisory_set(groups, package, relation=None): - from vulnerabilities.models import AdvisorySet - from vulnerabilities.models import AdvisorySetMember - from vulnerabilities.models import Group AdvisorySet.objects.filter(package=package, relation_type=relation).delete() @@ -50,3 +52,54 @@ def delete_and_save_advisory_set(groups, package, relation=None): ) AdvisorySetMember.objects.bulk_create(membership_to_create) + + +def group_advisory_for_package(package, logger=None): + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + + fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + + try: + affected_groups: List[Group] = merge_advisories(affecting_advisories, package) + fixed_by_groups: List[Group] = merge_advisories(fixed_by_advisories, package) + delete_and_save_advisory_set(affected_groups, package, relation="affecting") + delete_and_save_advisory_set(fixed_by_groups, package, relation="fixing") + logger(f"Successfully rebuilt advisory sets for package {package.purl}") + except Exception as e: + logger(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") + return + +def merge_advisories(advisories, package): + """ + Merge advisories based on their content hash and identifiers. + """ + from vulnerabilities.utils import compute_advisory_content_hash + from vulnerabilities.utils import get_merged_identifier_groups + + advisories = list(advisories) + + content_hash_map = defaultdict(list) + + for adv in advisories: + content_hash = compute_advisory_content_hash(adv, package) + content_hash_map[content_hash].append(adv) + + final_groups: List[Group] = [] + + for group in content_hash_map.values(): + groups = get_merged_identifier_groups(group) + final_groups.extend(groups) + + return final_groups diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 48d3b5384..86b6919ad 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -44,7 +44,6 @@ from univers.version_range import VersionRange from aboutcode.hashid import build_vcid -from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set logger = logging.getLogger(__name__) @@ -869,29 +868,6 @@ def compute_patch_checksum(patch_text: str): return hashlib.sha512(patch_text.encode("utf-8")).hexdigest() -def merge_advisories(advisories, package): - """ - Merge advisories based on their content hash and identifiers. - """ - from vulnerabilities.models import Group - - advisories = list(advisories) - - content_hash_map = defaultdict(list) - - for adv in advisories: - content_hash = compute_advisory_content_hash(adv, package) - content_hash_map[content_hash].append(adv) - - final_groups: List[Group] = [] - - for group in content_hash_map.values(): - groups = get_merged_identifier_groups(group) - final_groups.extend(groups) - - return final_groups - - def compute_advisory_content_hash(adv, package): """Compute a content hash for an advisory based on its affected and fixed packages for a given package. This is used to determine if two advisories are the same based on their content.""" diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 3aff06768..d7a23ffab 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -390,41 +390,41 @@ def get_context_data(self, **kwargs): return context - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( - purl=package.purl - ) - - fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( - purl=package.purl - ) - fixed_pkg_details = get_fixed_package_details(package) - context["fixed_package_details"] = fixed_pkg_details - context["grouped"] = True - - affecting_advisories = affecting_advisories.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - affected_by_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, affecting_advisories, "affecting" - ) - - fixed_by_advisories = fixed_by_advisories.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - fixing_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, fixed_by_advisories, "fixing" - ) - - context["affected_by_advisories_v2"] = affected_by_advisories - context["fixing_advisories_v2"] = fixing_advisories - return context + # if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + # affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + # purl=package.purl + # ) + + # fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + # purl=package.purl + # ) + # fixed_pkg_details = get_fixed_package_details(package) + # context["fixed_package_details"] = fixed_pkg_details + # context["grouped"] = True + + # affecting_advisories = affecting_advisories.prefetch_related( + # "aliases", + # "impacted_packages__affecting_packages", + # "impacted_packages__fixed_by_packages", + # ) + + # affected_by_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( + # package, affecting_advisories, "affecting" + # ) + + # fixed_by_advisories = fixed_by_advisories.prefetch_related( + # "aliases", + # "impacted_packages__affecting_packages", + # "impacted_packages__fixed_by_packages", + # ) + + # fixing_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( + # package, fixed_by_advisories, "fixing" + # ) + + # context["affected_by_advisories_v2"] = affected_by_advisories + # context["fixing_advisories_v2"] = fixing_advisories + # return context def get_object(self, queryset=None): if queryset is None: From 68972109d65a23264a20871821dd8d455c13ed89 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 27 May 2026 18:11:37 +0530 Subject: [PATCH 09/24] Calculate risk score at time of advisory insertion Signed-off-by: Tushar Goel --- .../v2_improvers/unfurl_version_range.py | 13 +++- vulnerabilities/pipes/advisory.py | 50 +++++++++++- vulnerabilities/pipes/group_advisories.py | 15 +++- vulnerabilities/pipes/risk_score.py | 77 +++++++++++++++++++ 4 files changed, 147 insertions(+), 8 deletions(-) create mode 100644 vulnerabilities/pipes/risk_score.py diff --git a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py index 3540b45b7..6e8831906 100644 --- a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py +++ b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py @@ -12,9 +12,9 @@ from traceback import format_exc as traceback_format_exc from aboutcode.pipeline import LoopProgress +from django.db import transaction from django.db.models import F from django.db.models import Q -from django.db import transaction from django.utils import timezone from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS as FETCHCODE_SUPPORTED_ECOSYSTEMS from packageurl import PackageURL @@ -28,6 +28,7 @@ from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipes.fetchcode_utils import get_versions from vulnerabilities.pipes.group_advisories import group_advisory_for_package +from vulnerabilities.pipes.risk_score import compute_package_risk_score from vulnerabilities.utils import update_purl_version @@ -178,15 +179,19 @@ def bulk_create_with_m2m(purls, impact, relation, logger): relation(impacted_package=impact, package=package) for package in affected_packages_v2 ] - for pkg in affected_packages_v2: - group_advisory_for_package(pkg, logger=logger) - try: relation.objects.bulk_create(relations, ignore_conflicts=True) except Exception as e: logger(f"Error creating ImpactedPackage {relation}: {e!r} \n {traceback_format_exc()}") return 0 + for pkg in affected_packages_v2: + group_advisory_for_package(pkg, logger=logger) + risk_score = compute_package_risk_score(pkg) + logger(f"Computed risk score {risk_score} for package {pkg.purl}") + pkg.risk_score = risk_score + pkg.save() + return len(relations) diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index d4f45a38c..5ec920969 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -48,6 +48,8 @@ from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness from vulnerabilities.pipes.group_advisories import group_advisory_for_package +from vulnerabilities.pipes.risk_score import compute_advisory_risk_score +from vulnerabilities.pipes.risk_score import compute_package_risk_score from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 @@ -363,6 +365,12 @@ def insert_advisory_v2( if values: getattr(advisory_obj, field_name).add(*values) + weighted_severity, exploitability, risk_score = compute_advisory_risk_score(advisory_obj) + advisory_obj.weighted_severity = round(weighted_severity, 1) if weighted_severity is not None else None + advisory_obj.exploitability = round(exploitability, 1) if exploitability is not None else None + advisory_obj.risk_score = round(risk_score, 1) if risk_score is not None else None + advisory_obj.save() + for affected_pkg in advisory.affected_packages: impact = ImpactedPackage.objects.create( advisory=advisory_obj, @@ -388,9 +396,47 @@ def insert_advisory_v2( purls=package_fixed_purls ) - for pkg in list(affected_packages_v2) + list(fixed_packages_v2): + for pkg in list(affected_packages_v2): + logger(f"Grouping advisories for package: {pkg.purl}") + try: + group_advisory_for_package( + pkg, + logger=logger, + current_advisory=advisory_obj, + current_advisory_relation="affecting", + ) + except Exception as e: + logger( + f"Failed to group advisories for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + try: + risk_score = compute_package_risk_score(pkg, current_advisory_risk_score=advisory_obj.risk_score) + logger(f"Computed risk score {risk_score} for package {pkg.purl}") + pkg.risk_score = risk_score + pkg.save() + pkg.calculate_version_rank + except Exception as e: + logger( + f"Failed to compute risk score for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + + for pkg in list(fixed_packages_v2): logger(f"Grouping advisories for package: {pkg.purl}") - group_advisory_for_package(pkg, logger=logger) + try: + group_advisory_for_package( + pkg, + logger=logger, + current_advisory=advisory_obj, + current_advisory_relation="fixing", + ) + except Exception as e: + logger( + f"Failed to group advisories for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + pkg.calculate_version_rank impact.affecting_packages.add(*affected_packages_v2) impact.fixed_by_packages.add(*fixed_packages_v2) diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py index d72f0872b..04093a12d 100644 --- a/vulnerabilities/pipes/group_advisories.py +++ b/vulnerabilities/pipes/group_advisories.py @@ -11,7 +11,10 @@ from typing import List from django.db import transaction -from vulnerabilities.models import AdvisorySet, AdvisoryV2, AdvisorySetMember + +from vulnerabilities.models import AdvisorySet +from vulnerabilities.models import AdvisorySetMember +from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import Group @@ -54,7 +57,9 @@ def delete_and_save_advisory_set(groups, package, relation=None): AdvisorySetMember.objects.bulk_create(membership_to_create) -def group_advisory_for_package(package, logger=None): +def group_advisory_for_package( + package, logger=None, current_advisory=None, current_advisory_relation=None +): affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( purl=package.purl ).prefetch_related( @@ -71,6 +76,11 @@ def group_advisory_for_package(package, logger=None): "impacted_packages__fixed_by_packages", ) + if current_advisory and current_advisory_relation == "affecting": + affecting_advisories = list(affecting_advisories) + [current_advisory] + elif current_advisory and current_advisory_relation == "fixing": + fixed_by_advisories = list(fixed_by_advisories) + [current_advisory] + try: affected_groups: List[Group] = merge_advisories(affecting_advisories, package) fixed_by_groups: List[Group] = merge_advisories(fixed_by_advisories, package) @@ -81,6 +91,7 @@ def group_advisory_for_package(package, logger=None): logger(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") return + def merge_advisories(advisories, package): """ Merge advisories based on their content hash and identifiers. diff --git a/vulnerabilities/pipes/risk_score.py b/vulnerabilities/pipes/risk_score.py new file mode 100644 index 000000000..24cc19a5b --- /dev/null +++ b/vulnerabilities/pipes/risk_score.py @@ -0,0 +1,77 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.db.models import Max + +from vulnerabilities.models import AdvisoryV2 + +from decimal import Decimal, ROUND_HALF_UP + +def quantize_1(value): + if value is None: + return None + + return Decimal(str(value)).quantize( + Decimal("0.1"), + rounding=ROUND_HALF_UP, + ) + + +def compute_package_risk_score(package, current_advisory_risk_score=None): + """Calculate the risk score for a single PackageV2 object.""" + max_risk = ( + AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) + .aggregate(max_risk=Max("risk_score")) + .get("max_risk") + ) + # include current advisory risk score in the calculation if provided and is higher than the max risk score from the database + if current_advisory_risk_score is not None: + max_risk = max(max_risk or 0, current_advisory_risk_score) + if max_risk is None: + return None + return round(float(max_risk), 1) + + +def compute_advisory_risk_score(advisory): + """ + Calculate the risk score for a single AdvisoryV2 object. + Returns a tuple of (weighted_severity, exploitability, risk_score). + """ + from vulnerabilities.risk import compute_vulnerability_risk_factors + + weighted_severity = None + exploitability = None + risk_score = None + + references = advisory.references.all() + exploits = advisory.exploits.all() + + severities = list(advisory.severities.all()) + + for rel in advisory.related_advisory_severities.all(): + severities.extend(rel.severities.all()) + + try: + calculated_weighted_severity, calculated_exploitability = ( + compute_vulnerability_risk_factors( + references=references, + severities=severities, + exploits=exploits, + ) + ) + + weighted_severity = calculated_weighted_severity + exploitability = calculated_exploitability + if exploitability and weighted_severity: + risk_score = min(float(exploitability * weighted_severity), 10.0) + risk_score = round(risk_score, 1) + except Exception as e: + risk_score = None + + return quantize_1(weighted_severity), quantize_1(exploitability), quantize_1(risk_score) From 09943a8ab64d63827ad00dee54959b527653f2e4 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 27 May 2026 18:17:18 +0530 Subject: [PATCH 10/24] Add type check for grouping of individual package Signed-off-by: Tushar Goel --- vulnerabilities/pipes/group_advisories.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py index 04093a12d..55672d43b 100644 --- a/vulnerabilities/pipes/group_advisories.py +++ b/vulnerabilities/pipes/group_advisories.py @@ -60,6 +60,14 @@ def delete_and_save_advisory_set(groups, package, relation=None): def group_advisory_for_package( package, logger=None, current_advisory=None, current_advisory_relation=None ): + """ + Group advisories for a given package and save the advisory sets for the package. + """ + from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS + + if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: + return + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( purl=package.purl ).prefetch_related( From 2ac09422181316e0547ec2470e0599a7b2a86f26 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 27 May 2026 18:22:12 +0530 Subject: [PATCH 11/24] Add Changelog Signed-off-by: Tushar Goel --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f701c0802..9960233e2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,8 @@ next release --------------------- - WARNING: Vulnerablecode V1 API and UI has stopped supporting Ubuntu OVAL advisories, please shift to V3 API for new Ubuntu advisories. +- WARNING: We will deprecate improver pipelines for calculating package version rank, grouping advisories for packages and calculating risk scores in the next release, we are doing it at advisory import time instead of as separate pipelines, this will improve the performance and consistency of the data. +- Calculate package verion rank, group advisories for packages and package risk score and advisory risk score during import of advisories. - Add attribute ``pipeline_id`` to AdvisoryV2 to track the pipeline that created the advisory, also rename existing ``datasource_id`` and AVIDs. Version v38.6.0 From 6747562fb533d3afaaafeb1df06a58127f4f58ea Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 27 May 2026 20:25:38 +0530 Subject: [PATCH 12/24] Remove group advisories and version rank pipeline Signed-off-by: Tushar Goel --- vulnerabilities/improvers/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 5b47a7cf1..b1db301d4 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -11,7 +11,6 @@ from vulnerabilities.improvers import vulnerability_status from vulnerabilities.pipelines import add_cvss31_to_CVEs from vulnerabilities.pipelines import compute_package_risk -from vulnerabilities.pipelines import compute_package_version_rank from vulnerabilities.pipelines import enhance_with_exploitdb from vulnerabilities.pipelines import enhance_with_kev from vulnerabilities.pipelines import enhance_with_metasploit @@ -32,7 +31,6 @@ enhance_with_metasploit as enhance_with_metasploit_v2, ) from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2 -from vulnerabilities.pipelines.v2_improvers import group_advisories_for_packages from vulnerabilities.pipelines.v2_improvers import reference_collect_commits from vulnerabilities.pipelines.v2_improvers import relate_severities from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2 @@ -61,7 +59,6 @@ enhance_with_metasploit.MetasploitImproverPipeline, enhance_with_exploitdb.ExploitDBImproverPipeline, compute_package_risk.ComputePackageRiskPipeline, - compute_package_version_rank.ComputeVersionRankPipeline, add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline, remove_duplicate_advisories.RemoveDuplicateAdvisoriesPipeline, populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline, @@ -75,7 +72,6 @@ collect_ssvc_trees.CollectSSVCPipeline, relate_severities.RelateSeveritiesPipeline, archive_urls.ArchiveImproverPipeline, - group_advisories_for_packages.GroupAdvisoriesForPackages, compute_advisory_todo_v2.ComputeToDo, reference_collect_commits.CollectReferencesFixCommitsPipeline, enhance_with_github_poc.GithubPocsImproverPipeline, From 887602e150f159b49c57cda804ac98334837e2b9 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 27 May 2026 20:28:50 +0530 Subject: [PATCH 13/24] Remove group advisories and version rank pipeline Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index beb65f91e..973db8132 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -236,7 +236,7 @@ def get_affected_by_vulnerabilities(self, package): result.append( { - "advisory_id": adv["identifier"], + "advisory_id": adv["advisory_id"], "advisory_uid": adv["advisory_uid"], "aliases": adv["aliases"], "summary": adv["summary"], From c6ba1b1afb91b9eb1aff9efd8914a6e0f7fa4be2 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 28 May 2026 14:06:49 +0530 Subject: [PATCH 14/24] Remove empty package_urls Signed-off-by: Tushar Goel --- .../v2_improvers/group_advisories_for_packages.py | 1 + vulnerabilities/utils.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index b85c2d538..642b5e3d2 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -17,6 +17,7 @@ class GroupAdvisoriesForPackages(VulnerableCodePipeline): """Group advisories for packages that have multiple importers""" pipeline_id = "group_advisories_for_packages" + run_once = True @classmethod def steps(cls): diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 86b6919ad..0b313974c 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -883,8 +883,12 @@ def compute_advisory_content_hash(adv, package): ) for impact in adv.impacted_packages.filter(base_purl=str(version_less_purl)): - affected.extend([pkg.package_url for pkg in impact.affecting_packages.all()]) - fixed.extend([pkg.package_url for pkg in impact.fixed_by_packages.all()]) + for pkg in impact.affecting_packages.all(): + if pkg.package_url: + affected.extend([pkg.package_url]) + for pkg in impact.fixed_by_packages.all(): + if pkg.package_url: + fixed.extend([pkg.package_url]) normalized_data = { "affected_packages": normalize_list(affected), From c592e88d82322059bbb499677cf8895f1a3d92f6 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 28 May 2026 19:31:54 +0530 Subject: [PATCH 15/24] Add migration Signed-off-by: Tushar Goel --- .../0134_alter_advisoryset_unique_together.py | 17 +++++++++++++++++ vulnerabilities/pipes/advisory.py | 6 +++--- vulnerabilities/pipes/group_advisories.py | 3 ++- 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 vulnerabilities/migrations/0134_alter_advisoryset_unique_together.py diff --git a/vulnerabilities/migrations/0134_alter_advisoryset_unique_together.py b/vulnerabilities/migrations/0134_alter_advisoryset_unique_together.py new file mode 100644 index 000000000..7b0c2a4be --- /dev/null +++ b/vulnerabilities/migrations/0134_alter_advisoryset_unique_together.py @@ -0,0 +1,17 @@ +# Generated by Django 5.2.11 on 2026-05-28 13:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more"), + ] + + operations = [ + migrations.AlterUniqueTogether( + name="advisoryset", + unique_together={("package", "relation_type", "primary_advisory")}, + ), + ] diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 5ec920969..21ca7b996 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -396,6 +396,9 @@ def insert_advisory_v2( purls=package_fixed_purls ) + impact.affecting_packages.add(*affected_packages_v2) + impact.fixed_by_packages.add(*fixed_packages_v2) + for pkg in list(affected_packages_v2): logger(f"Grouping advisories for package: {pkg.purl}") try: @@ -438,9 +441,6 @@ def insert_advisory_v2( ) pkg.calculate_version_rank - impact.affecting_packages.add(*affected_packages_v2) - impact.fixed_by_packages.add(*fixed_packages_v2) - introduced_commit_v2 = get_or_create_advisory_package_commit_patches( affected_pkg.introduced_by_commit_patches ) diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py index 55672d43b..0f8981e45 100644 --- a/vulnerabilities/pipes/group_advisories.py +++ b/vulnerabilities/pipes/group_advisories.py @@ -96,7 +96,8 @@ def group_advisory_for_package( delete_and_save_advisory_set(fixed_by_groups, package, relation="fixing") logger(f"Successfully rebuilt advisory sets for package {package.purl}") except Exception as e: - logger(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") + if logger: + logger(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") return From a682a8f6da81fac3a070f8da77d6b5209ff1a4c7 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 29 May 2026 12:47:59 +0530 Subject: [PATCH 16/24] Stablizie grouping and risk score calculation Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 33 +---- vulnerabilities/models.py | 3 + .../v2_improvers/unfurl_version_range.py | 43 +++++- vulnerabilities/pipes/advisory.py | 122 +++++++++++------- vulnerabilities/pipes/group_advisories.py | 53 ++++---- vulnerabilities/pipes/risk_score.py | 4 +- 6 files changed, 155 insertions(+), 103 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 973db8132..7d1251cac 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -251,11 +251,13 @@ def get_affected_by_vulnerabilities(self, package): return result - advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) + if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: + advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + package.package_url + ) - advisories = [] + advisories = [] - if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: advisories_ids = advisories_qs.only("id") advisories_ids = list(advisories_ids[:101]) @@ -322,18 +324,6 @@ def get_affected_by_vulnerabilities(self, package): return result - # if not advisories: - # if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - # advisories_qs = advisories_qs.prefetch_related( - # "aliases", - # "impacted_packages__affecting_packages", - # "impacted_packages__fixed_by_packages", - # ) - # advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - # package, advisories_qs, "affecting" - # ) - # return self.return_advisories_data(package, advisories_qs, advisories) - def get_fixing_vulnerabilities(self, package): advisories = self.context["fixing_advisory_map"].get(package.id, []) results = [] @@ -375,19 +365,6 @@ def get_fixing_vulnerabilities(self, package): ) return results - # if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - # advisories_qs = advisories_qs.prefetch_related( - # "aliases", - # "impacted_packages__affecting_packages", - # "impacted_packages__fixed_by_packages", - # ) - # if not advisories_qs.exists(): - # return [] - # advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - # package, advisories_qs, "fixing" - # ) - # return self.return_fixing_advisories_data(advisories) - def return_fixing_advisories_data(self, advisories): result = [] for advisory in advisories: diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 8ed6d2520..1a98f3bc9 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -3034,6 +3034,9 @@ class AdvisorySet(models.Model): created_at = models.DateTimeField(auto_now_add=True) + class Meta: + unique_together = ("package", "relation_type", "primary_advisory") + class AdvisorySetMember(models.Model): diff --git a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py index 6e8831906..866ae50fc 100644 --- a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py +++ b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py @@ -173,22 +173,51 @@ def bulk_create_with_m2m(purls, impact, relation, logger): if not affected_packages_v2.exists(): return 0 - affected_packages_v2.first().calculate_version_rank + affected_packages_v2 = sorted( + affected_packages_v2, + key=lambda p: p.purl, + ) + + locked_packages = [] + + for pkg in affected_packages_v2: + locked_pkg = PackageV2.objects.select_for_update().get(id=pkg.id) + + locked_packages.append(locked_pkg) + + locked_packages[0].calculate_version_rank relations = [ - relation(impacted_package=impact, package=package) for package in affected_packages_v2 + relation( + impacted_package=impact, + package=package, + ) + for package in locked_packages ] try: - relation.objects.bulk_create(relations, ignore_conflicts=True) + relation.objects.bulk_create( + relations, + ignore_conflicts=True, + ) + except Exception as e: - logger(f"Error creating ImpactedPackage {relation}: {e!r} \n {traceback_format_exc()}") + logger( + f"Error creating ImpactedPackage " f"{relation}: {e!r} \n " f"{traceback_format_exc()}" + ) + return 0 + for pkg in locked_packages: + + group_advisory_for_package( + pkg, + logger=logger, + ) - for pkg in affected_packages_v2: - group_advisory_for_package(pkg, logger=logger) risk_score = compute_package_risk_score(pkg) - logger(f"Computed risk score {risk_score} for package {pkg.purl}") + + logger(f"Computed risk score {risk_score} " f"for package {pkg.purl}") + pkg.risk_score = risk_score pkg.save() diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 21ca7b996..8eea36d1d 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -366,7 +366,9 @@ def insert_advisory_v2( getattr(advisory_obj, field_name).add(*values) weighted_severity, exploitability, risk_score = compute_advisory_risk_score(advisory_obj) - advisory_obj.weighted_severity = round(weighted_severity, 1) if weighted_severity is not None else None + advisory_obj.weighted_severity = ( + round(weighted_severity, 1) if weighted_severity is not None else None + ) advisory_obj.exploitability = round(exploitability, 1) if exploitability is not None else None advisory_obj.risk_score = round(risk_score, 1) if risk_score is not None else None advisory_obj.save() @@ -396,50 +398,9 @@ def insert_advisory_v2( purls=package_fixed_purls ) - impact.affecting_packages.add(*affected_packages_v2) - impact.fixed_by_packages.add(*fixed_packages_v2) - - for pkg in list(affected_packages_v2): - logger(f"Grouping advisories for package: {pkg.purl}") - try: - group_advisory_for_package( - pkg, - logger=logger, - current_advisory=advisory_obj, - current_advisory_relation="affecting", - ) - except Exception as e: - logger( - f"Failed to group advisories for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", - level=logging.ERROR, - ) - try: - risk_score = compute_package_risk_score(pkg, current_advisory_risk_score=advisory_obj.risk_score) - logger(f"Computed risk score {risk_score} for package {pkg.purl}") - pkg.risk_score = risk_score - pkg.save() - pkg.calculate_version_rank - except Exception as e: - logger( - f"Failed to compute risk score for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", - level=logging.ERROR, - ) - - for pkg in list(fixed_packages_v2): - logger(f"Grouping advisories for package: {pkg.purl}") - try: - group_advisory_for_package( - pkg, - logger=logger, - current_advisory=advisory_obj, - current_advisory_relation="fixing", - ) - except Exception as e: - logger( - f"Failed to group advisories for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", - level=logging.ERROR, - ) - pkg.calculate_version_rank + add_packages_to_advisory( + logger, advisory_obj, impact, affected_packages_v2, fixed_packages_v2 + ) introduced_commit_v2 = get_or_create_advisory_package_commit_patches( affected_pkg.introduced_by_commit_patches @@ -452,6 +413,77 @@ def insert_advisory_v2( return advisory_obj +@transaction.atomic +def add_packages_to_advisory(logger, advisory_obj, impact, affected_packages_v2, fixed_packages_v2): + from vulnerabilities.models import PackageV2 + + all_packages = sorted( + { + *affected_packages_v2, + *fixed_packages_v2, + }, + key=lambda p: p.purl, + ) + + locked_packages = {} + + for pkg in all_packages: + locked_pkg = PackageV2.objects.select_for_update().get(id=pkg.id) + + locked_packages[pkg.id] = locked_pkg + + affected_packages_v2 = [locked_packages[pkg.id] for pkg in affected_packages_v2] + + fixed_packages_v2 = [locked_packages[pkg.id] for pkg in fixed_packages_v2] + + impact.affecting_packages.add(*affected_packages_v2) + impact.fixed_by_packages.add(*fixed_packages_v2) + + for pkg in list(affected_packages_v2): + logger(f"Grouping advisories for package: {pkg.purl}") + try: + group_advisory_for_package( + pkg, + logger=logger, + current_advisory=advisory_obj, + current_advisory_relation="affecting", + ) + except Exception as e: + logger( + f"Failed to group advisories for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + try: + risk_score = compute_package_risk_score( + pkg, current_advisory_risk_score=advisory_obj.risk_score + ) + logger(f"Computed risk score {risk_score} for package {pkg.purl}") + pkg.risk_score = risk_score + pkg.save() + pkg.calculate_version_rank + except Exception as e: + logger( + f"Failed to compute risk score for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + + for pkg in list(fixed_packages_v2): + logger(f"Grouping advisories for package: {pkg.purl}") + try: + group_advisory_for_package( + pkg, + logger=logger, + current_advisory=advisory_obj, + current_advisory_relation="fixing", + ) + except Exception as e: + logger( + f"Failed to group advisories for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + pkg.calculate_version_rank + + @transaction.atomic def import_advisory( advisory: Advisory, diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py index 0f8981e45..35cd07014 100644 --- a/vulnerabilities/pipes/group_advisories.py +++ b/vulnerabilities/pipes/group_advisories.py @@ -22,39 +22,46 @@ def delete_and_save_advisory_set(groups, package, relation=None): AdvisorySet.objects.filter(package=package, relation_type=relation).delete() - membership_to_create = [] + print(f"Grouping advisories for package: {package.purl}") + for group in groups: - assert isinstance(group, Group) - advisory_set = AdvisorySet.objects.create( - package=package, - relation_type=relation, - primary_advisory=group.primary, - ) - - advisory_set.aliases.add(*group.aliases) - advisory_set.save() - - membership_to_create.append( - AdvisorySetMember( - advisory_set=advisory_set, - advisory=group.primary, - is_primary=True, + try: + advisory_set = AdvisorySet.objects.get_or_create( + package=package, + relation_type=relation, + primary_advisory=group.primary, ) - ) - for adv in group.secondaries: + advisory_set.aliases.add(*group.aliases) + advisory_set.save() + membership_to_create.append( AdvisorySetMember( advisory_set=advisory_set, - advisory=adv, - is_primary=False, + advisory=group.primary, + is_primary=True, + ) + ) + + for adv in group.secondaries: + membership_to_create.append( + AdvisorySetMember( + advisory_set=advisory_set, + advisory=adv, + is_primary=False, + ) ) + except Exception as e: + print( + f"Failed to create advisory set for package {package.purl} with primary advisory {group.primary}: {e!r}" ) + continue AdvisorySetMember.objects.bulk_create(membership_to_create) + print(f"Successfully saved advisory sets for package: {package.purl}") def group_advisory_for_package( @@ -85,9 +92,11 @@ def group_advisory_for_package( ) if current_advisory and current_advisory_relation == "affecting": - affecting_advisories = list(affecting_advisories) + [current_advisory] + if not affecting_advisories.filter(id=current_advisory.id).exists(): + affecting_advisories = list(affecting_advisories) + [current_advisory] elif current_advisory and current_advisory_relation == "fixing": - fixed_by_advisories = list(fixed_by_advisories) + [current_advisory] + if not fixed_by_advisories.filter(id=current_advisory.id).exists(): + fixed_by_advisories = list(fixed_by_advisories) + [current_advisory] try: affected_groups: List[Group] = merge_advisories(affecting_advisories, package) diff --git a/vulnerabilities/pipes/risk_score.py b/vulnerabilities/pipes/risk_score.py index 24cc19a5b..8121bd6c2 100644 --- a/vulnerabilities/pipes/risk_score.py +++ b/vulnerabilities/pipes/risk_score.py @@ -7,11 +7,13 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from decimal import ROUND_HALF_UP +from decimal import Decimal + from django.db.models import Max from vulnerabilities.models import AdvisoryV2 -from decimal import Decimal, ROUND_HALF_UP def quantize_1(value): if value is None: From adbe227db25c2ad29420cd13018324b5b99a913c Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 29 May 2026 13:43:53 +0530 Subject: [PATCH 17/24] Fix errors Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 2 - vulnerabilities/pipes/group_advisories.py | 3 +- .../test_compute_advisory_todo_v2.py | 2 - vulnerabilities/tests/test_advisory_merge.py | 26 +------------ vulnerabilities/tests/test_api_v3.py | 2 +- vulnerabilities/tests/test_risk.py | 5 --- vulnerabilities/utils.py | 11 ------ vulnerabilities/views.py | 38 +------------------ 8 files changed, 5 insertions(+), 84 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 7d1251cac..b9e0d94dc 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -8,7 +8,6 @@ # from collections import defaultdict -from typing import List from urllib.parse import urlencode from django.db.models import Exists @@ -36,7 +35,6 @@ from vulnerabilities.models import PackageV2 from vulnerabilities.throttling import PermissionBasedUserRateThrottle from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS -from vulnerabilities.utils import merge_and_save_grouped_advisories class PackageQuerySerializer(serializers.Serializer): diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py index 35cd07014..d1266ceea 100644 --- a/vulnerabilities/pipes/group_advisories.py +++ b/vulnerabilities/pipes/group_advisories.py @@ -27,9 +27,8 @@ def delete_and_save_advisory_set(groups, package, relation=None): print(f"Grouping advisories for package: {package.purl}") for group in groups: - try: - advisory_set = AdvisorySet.objects.get_or_create( + advisory_set, _ = AdvisorySet.objects.get_or_create( package=package, relation_type=relation, primary_advisory=group.primary, diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py index a1bbe0465..3d4bfb219 100644 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py @@ -381,8 +381,6 @@ def test_todo_conflict_details_partial_curation(self): result_partial_curation = issue_details["partial_curation_advisory"] self.assertEqual(1, AdvisoryToDoV2.objects.count()) self.assertEqual("CONFLICTING_FIXED_BY_PACKAGES", todo.issue_type) - print(result_partial_curation) - # breakpoint() self.assertDictEqual(expected_partial_curation_advisory, result_partial_curation) def test_todo_conflict_details_partial_curation_unpaired_purl_and_conflicting_affected_and_fixed( diff --git a/vulnerabilities/tests/test_advisory_merge.py b/vulnerabilities/tests/test_advisory_merge.py index 71c214bbb..c4c738483 100644 --- a/vulnerabilities/tests/test_advisory_merge.py +++ b/vulnerabilities/tests/test_advisory_merge.py @@ -18,12 +18,11 @@ from vulnerabilities.models import Group from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PackageV2 +from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set +from vulnerabilities.pipes.group_advisories import merge_advisories from vulnerabilities.utils import compute_advisory_content_hash -from vulnerabilities.utils import delete_and_save_advisory_set from vulnerabilities.utils import get_advisories_from_groups from vulnerabilities.utils import get_merged_identifier_groups -from vulnerabilities.utils import merge_advisories -from vulnerabilities.utils import merge_and_save_grouped_advisories @pytest.mark.django_db @@ -162,27 +161,6 @@ def test_delete_and_save_advisory_set(self): assert any(m.is_primary for m in members) assert any(not m.is_primary for m in members) - def test_merge_and_save_integration(self): - package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") - - adv1 = self.create_advisory("A1", ["1.0"], ["2.0"]) - adv2 = self.create_advisory("A2", ["1.0"], ["2.0"]) - - alias = AdvisoryAlias.objects.create(alias="CVE-1") - - adv1.aliases.add(alias) - adv2.aliases.add(alias) - - result = merge_and_save_grouped_advisories( - package, - [adv1, adv2], - relation="test", - ) - - assert len(result) == 1 - assert AdvisorySet.objects.count() == 1 - assert AdvisorySetMember.objects.count() == 2 - def test_merge_advisories_separates_different_content(self): package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py index cd944a203..3a241361f 100644 --- a/vulnerabilities/tests/test_api_v3.py +++ b/vulnerabilities/tests/test_api_v3.py @@ -67,7 +67,7 @@ def test_packages_post_without_details(self): def test_packages_post_with_details(self): url = reverse("package-v3-list") - with self.assertNumQueries(31): + with self.assertNumQueries(12): response = self.client.post( url, data={ diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py index 420c8c402..b5c4d4997 100644 --- a/vulnerabilities/tests/test_risk.py +++ b/vulnerabilities/tests/test_risk.py @@ -24,7 +24,6 @@ @pytest.fixture -@pytest.mark.django_db def vulnerability(): vul = Vulnerability(vulnerability_id="VCID-Existing") vul.save() @@ -50,7 +49,6 @@ def vulnerability(): @pytest.fixture -@pytest.mark.django_db def exploit(): vul = Vulnerability(vulnerability_id="VCID-Exploit") vul.save() @@ -58,7 +56,6 @@ def exploit(): @pytest.fixture -@pytest.mark.django_db def vulnerability_with_exploit_ref(): vul = Vulnerability(vulnerability_id="VCID-Exploit-Ref") vul.save() @@ -74,7 +71,6 @@ def vulnerability_with_exploit_ref(): @pytest.fixture -@pytest.mark.django_db def high_epss_score(): vul = Vulnerability(vulnerability_id="VCID-HIGH-EPSS") vul.save() @@ -90,7 +86,6 @@ def high_epss_score(): @pytest.fixture -@pytest.mark.django_db def low_epss_score(): vul = Vulnerability(vulnerability_id="VCID-LOW-EPSS") vul.save() diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 0b313974c..dc72ad4d2 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -1043,17 +1043,6 @@ def get_advisories_from_groups(groups, include_ssvc_trees=False): return advisories -def merge_and_save_grouped_advisories(package, advisories, relation): - """ - Merge advisories based on their content and identifiers and save the merged advisories to the database. - """ - groups = merge_advisories(advisories, package) - delete_and_save_advisory_set(groups, package, relation) - advisories = get_advisories_from_groups(groups) - - return advisories - - TYPES_WITH_MULTIPLE_IMPORTERS = [ "pypi", "maven", diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index d7a23ffab..88d9c8182 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -53,7 +53,6 @@ from vulnerabilities.throttling import AnonUserUIThrottle from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS from vulnerabilities.utils import get_advisories_from_groups -from vulnerabilities.utils import merge_and_save_grouped_advisories from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerablecode.settings import env @@ -327,6 +326,7 @@ def get_context_data(self, **kwargs): return context is_grouped = models.AdvisorySet.objects.filter(package=package).exists() + print(f"Is package {package.purl} grouped? {is_grouped}") if is_grouped: context["grouped"] = True @@ -390,42 +390,6 @@ def get_context_data(self, **kwargs): return context - # if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - # affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( - # purl=package.purl - # ) - - # fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( - # purl=package.purl - # ) - # fixed_pkg_details = get_fixed_package_details(package) - # context["fixed_package_details"] = fixed_pkg_details - # context["grouped"] = True - - # affecting_advisories = affecting_advisories.prefetch_related( - # "aliases", - # "impacted_packages__affecting_packages", - # "impacted_packages__fixed_by_packages", - # ) - - # affected_by_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - # package, affecting_advisories, "affecting" - # ) - - # fixed_by_advisories = fixed_by_advisories.prefetch_related( - # "aliases", - # "impacted_packages__affecting_packages", - # "impacted_packages__fixed_by_packages", - # ) - - # fixing_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - # package, fixed_by_advisories, "fixing" - # ) - - # context["affected_by_advisories_v2"] = affected_by_advisories - # context["fixing_advisories_v2"] = fixing_advisories - # return context - def get_object(self, queryset=None): if queryset is None: queryset = self.get_queryset() From 7abc825018c73aacac05c2df73a4dcf41402d1af Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 29 May 2026 14:14:00 +0530 Subject: [PATCH 18/24] Fix tests Signed-off-by: Tushar Goel --- .../test_compute_advisory_todo_v2.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py index 3d4bfb219..1c6de8e4d 100644 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py @@ -293,13 +293,13 @@ def test_todo_conflict_details_partial_curation(self): "package": { "type": "npm", "namespace": "", - "name": "package1", + "name": "package2", "version": "", "qualifiers": "", "subpath": "", }, "affected_version_range": "vers:npm/>=1.0.0|<=2.0.0", - "fixed_version_range": "vers:npm/2.0.1", + "fixed_version_range": None, "introduced_by_commit_patches": [], "fixed_by_commit_patches": [], }, @@ -312,36 +312,36 @@ def test_todo_conflict_details_partial_curation(self): "qualifiers": "", "subpath": "", }, - "affected_version_range": "vers:npm/>=1.0.0|<=2.0.0", + "affected_version_range": "vers:npm/>=3.0.0|<=3.9.0", "fixed_version_range": None, "introduced_by_commit_patches": [], "fixed_by_commit_patches": [], }, { "package": { - "type": "npm", + "type": "pypi", "namespace": "", - "name": "package2", + "name": "package1", "version": "", "qualifiers": "", "subpath": "", }, - "affected_version_range": "vers:npm/>=3.0.0|<=3.9.0", - "fixed_version_range": None, + "affected_version_range": "vers:pypi/>=1.0.0|<=2.0.0", + "fixed_version_range": "vers:pypi/2.0.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [], }, { "package": { - "type": "pypi", + "type": "npm", "namespace": "", "name": "package1", "version": "", "qualifiers": "", "subpath": "", }, - "affected_version_range": "vers:pypi/>=1.0.0|<=2.0.0", - "fixed_version_range": "vers:pypi/2.0.1", + "affected_version_range": "vers:npm/>=1.0.0|<=2.0.0", + "fixed_version_range": "vers:npm/2.0.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [], }, @@ -381,6 +381,7 @@ def test_todo_conflict_details_partial_curation(self): result_partial_curation = issue_details["partial_curation_advisory"] self.assertEqual(1, AdvisoryToDoV2.objects.count()) self.assertEqual("CONFLICTING_FIXED_BY_PACKAGES", todo.issue_type) + print(result_partial_curation) self.assertDictEqual(expected_partial_curation_advisory, result_partial_curation) def test_todo_conflict_details_partial_curation_unpaired_purl_and_conflicting_affected_and_fixed( From 2e18e6fd2bd9b2e41f1704e016e90d82f922c72c Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 29 May 2026 21:56:36 +0530 Subject: [PATCH 19/24] Speed up import process Signed-off-by: Tushar Goel --- .../v2_improvers/compute_package_risk.py | 12 +--- vulnerabilities/pipes/advisory.py | 52 ++++++------------ vulnerabilities/pipes/risk_score.py | 55 +++++++++++++++++++ vulnerabilities/views.py | 1 - 4 files changed, 74 insertions(+), 46 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index dacf7e6c8..1a93ba660 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -16,6 +16,7 @@ from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.pipes.risk_score import bulk_update from vulnerabilities.risk import compute_vulnerability_risk_factors @@ -178,14 +179,3 @@ def compute_and_store_package_risk_score(self): ) self.log(f"Successfully added risk score for {updated:,d} package") - -def bulk_update(model, items, fields, logger): - item_count = 0 - if items: - try: - model.objects.bulk_update(objs=items, fields=fields) - item_count += len(items) - except Exception as e: - logger(f"Error updating {model.__name__}: {e}") - items.clear() - return item_count diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 8eea36d1d..75a88b6da 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -48,7 +48,7 @@ from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness from vulnerabilities.pipes.group_advisories import group_advisory_for_package -from vulnerabilities.pipes.risk_score import compute_advisory_risk_score +from vulnerabilities.pipes.risk_score import compute_advisory_risk_score, compute_package_risk_score_bulk from vulnerabilities.pipes.risk_score import compute_package_risk_score from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 @@ -300,6 +300,7 @@ def insert_advisory_v2( ): from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PackageV2 + from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS from vulnerabilities.utils import compute_content_id_v2 advisory_obj = None @@ -398,9 +399,19 @@ def insert_advisory_v2( purls=package_fixed_purls ) - add_packages_to_advisory( - logger, advisory_obj, impact, affected_packages_v2, fixed_packages_v2 - ) + if affected_pkg.package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + group_advisories_for_package_with_lock( + advisory_obj, impact, affected_packages_v2, fixed_packages_v2, logger=logger + ) + else: + impact.affecting_packages.add(*affected_packages_v2) + impact.fixed_by_packages.add(*fixed_packages_v2) + + compute_package_risk_score_bulk(affected_packages_v2) + if affected_packages_v2: + affected_packages_v2[0].calculate_version_rank + elif fixed_packages_v2: + fixed_packages_v2[0].calculate_version_rank introduced_commit_v2 = get_or_create_advisory_package_commit_patches( affected_pkg.introduced_by_commit_patches @@ -414,7 +425,9 @@ def insert_advisory_v2( @transaction.atomic -def add_packages_to_advisory(logger, advisory_obj, impact, affected_packages_v2, fixed_packages_v2): +def group_advisories_for_package_with_lock( + advisory_obj, impact, affected_packages_v2, fixed_packages_v2, logger=print +): from vulnerabilities.models import PackageV2 all_packages = sorted( @@ -453,35 +466,6 @@ def add_packages_to_advisory(logger, advisory_obj, impact, affected_packages_v2, f"Failed to group advisories for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", level=logging.ERROR, ) - try: - risk_score = compute_package_risk_score( - pkg, current_advisory_risk_score=advisory_obj.risk_score - ) - logger(f"Computed risk score {risk_score} for package {pkg.purl}") - pkg.risk_score = risk_score - pkg.save() - pkg.calculate_version_rank - except Exception as e: - logger( - f"Failed to compute risk score for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", - level=logging.ERROR, - ) - - for pkg in list(fixed_packages_v2): - logger(f"Grouping advisories for package: {pkg.purl}") - try: - group_advisory_for_package( - pkg, - logger=logger, - current_advisory=advisory_obj, - current_advisory_relation="fixing", - ) - except Exception as e: - logger( - f"Failed to group advisories for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", - level=logging.ERROR, - ) - pkg.calculate_version_rank @transaction.atomic diff --git a/vulnerabilities/pipes/risk_score.py b/vulnerabilities/pipes/risk_score.py index 8121bd6c2..180897ff4 100644 --- a/vulnerabilities/pipes/risk_score.py +++ b/vulnerabilities/pipes/risk_score.py @@ -13,6 +13,7 @@ from django.db.models import Max from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 def quantize_1(value): @@ -40,6 +41,47 @@ def compute_package_risk_score(package, current_advisory_risk_score=None): return round(float(max_risk), 1) +def compute_package_risk_score_bulk(packages): + """Calculate the risk score for a single PackageV2 object.""" + purls = packages.values_list("package_url", flat=True) + advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purls(purls).only( + "id", "risk_score" + ) + qs = ( + PackageV2.objects.filter( + id__in=packages.values_list("id", flat=True), + affected_in_impacts__advisory__risk_score__isnull=False, + affected_in_impacts__advisory__in=advisories, + ) + .distinct() + .annotate(computed_risk=Max("affected_in_impacts__advisory__risk_score")) + .only("id") + ) + + batch = [] + batch_size = 5000 + updated = 0 + + for pkg in qs.iterator(chunk_size=batch_size): + pkg.risk_score = round(float(pkg.computed_risk), 1) + batch.append(pkg) + + if len(batch) >= batch_size: + updated += bulk_update( + model=PackageV2, + items=batch, + fields=["risk_score"], + ) + batch.clear() + + updated += bulk_update( + model=PackageV2, + items=batch, + fields=["risk_score"], + ) + + + def compute_advisory_risk_score(advisory): """ Calculate the risk score for a single AdvisoryV2 object. @@ -77,3 +119,16 @@ def compute_advisory_risk_score(advisory): risk_score = None return quantize_1(weighted_severity), quantize_1(exploitability), quantize_1(risk_score) + + +def bulk_update(model, items, fields, logger=None): + item_count = 0 + if items: + try: + model.objects.bulk_update(objs=items, fields=fields) + item_count += len(items) + except Exception as e: + if logger: + logger(f"Error updating {model.__name__}: {e}") + items.clear() + return item_count diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 88d9c8182..ea00bdc99 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -326,7 +326,6 @@ def get_context_data(self, **kwargs): return context is_grouped = models.AdvisorySet.objects.filter(package=package).exists() - print(f"Is package {package.purl} grouped? {is_grouped}") if is_grouped: context["grouped"] = True From 2ea35259841e4fa8389b51e41b0afc66fd6f8d81 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 29 May 2026 22:10:04 +0530 Subject: [PATCH 20/24] Fix formatting errors Signed-off-by: Tushar Goel --- .../pipelines/v2_improvers/compute_package_risk.py | 1 - vulnerabilities/pipes/advisory.py | 6 +++--- vulnerabilities/pipes/risk_score.py | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index 1a93ba660..3d3da7c94 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -178,4 +178,3 @@ def compute_and_store_package_risk_score(self): logger=self.log, ) self.log(f"Successfully added risk score for {updated:,d} package") - diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 75a88b6da..4f9fa8f55 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -48,8 +48,8 @@ from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness from vulnerabilities.pipes.group_advisories import group_advisory_for_package -from vulnerabilities.pipes.risk_score import compute_advisory_risk_score, compute_package_risk_score_bulk -from vulnerabilities.pipes.risk_score import compute_package_risk_score +from vulnerabilities.pipes.risk_score import compute_advisory_risk_score +from vulnerabilities.pipes.risk_score import compute_package_risk_score_bulk from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 @@ -411,7 +411,7 @@ def insert_advisory_v2( if affected_packages_v2: affected_packages_v2[0].calculate_version_rank elif fixed_packages_v2: - fixed_packages_v2[0].calculate_version_rank + fixed_packages_v2[0].calculate_version_rank introduced_commit_v2 = get_or_create_advisory_package_commit_patches( affected_pkg.introduced_by_commit_patches diff --git a/vulnerabilities/pipes/risk_score.py b/vulnerabilities/pipes/risk_score.py index 180897ff4..8a4bf00cf 100644 --- a/vulnerabilities/pipes/risk_score.py +++ b/vulnerabilities/pipes/risk_score.py @@ -79,7 +79,6 @@ def compute_package_risk_score_bulk(packages): items=batch, fields=["risk_score"], ) - def compute_advisory_risk_score(advisory): From d51561ba6f670e4c63e895544b2906ca08ce8b05 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sat, 30 May 2026 18:04:34 +0530 Subject: [PATCH 21/24] Fix grouping Signed-off-by: Tushar Goel --- vulnerabilities/pipes/advisory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 4f9fa8f55..286cf75dd 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -452,7 +452,7 @@ def group_advisories_for_package_with_lock( impact.affecting_packages.add(*affected_packages_v2) impact.fixed_by_packages.add(*fixed_packages_v2) - for pkg in list(affected_packages_v2): + for pkg in list(affected_packages_v2) + list(fixed_packages_v2): logger(f"Grouping advisories for package: {pkg.purl}") try: group_advisory_for_package( From c94a82f0477363852ee3002e65fb5361175d900f Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sat, 30 May 2026 18:24:05 +0530 Subject: [PATCH 22/24] Fasten up version range unfurling Signed-off-by: Tushar Goel --- .../v2_improvers/unfurl_version_range.py | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py index 866ae50fc..df0c092e8 100644 --- a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py +++ b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py @@ -29,6 +29,8 @@ from vulnerabilities.pipes.fetchcode_utils import get_versions from vulnerabilities.pipes.group_advisories import group_advisory_for_package from vulnerabilities.pipes.risk_score import compute_package_risk_score +from vulnerabilities.pipes.risk_score import compute_package_risk_score_bulk +from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS from vulnerabilities.utils import update_purl_version @@ -164,7 +166,9 @@ def get_purl_versions(purl, cached_versions, logger): @transaction.atomic def bulk_create_with_m2m(purls, impact, relation, logger): - """Bulk create PackageV2 and also bulk populate M2M Impact and Package relationships.""" + """Bulk create PackageV2 and also bulk populate M2M Impact and Package relationships. + This function assumes same base purl is used for all versions in ``purls`` list. + """ if not purls: return 0 @@ -173,6 +177,31 @@ def bulk_create_with_m2m(purls, impact, relation, logger): if not affected_packages_v2.exists(): return 0 + affected_packages_v2.first().calculate_version_rank + + if affected_packages_v2.first().type in TYPES_WITH_MULTIPLE_IMPORTERS: + relations = group_advisories_for_package_with_lock( + impact, affected_packages_v2, relation, logger + ) + + else: + relations = [ + relation(impacted_package=impact, package=package) for package in affected_packages_v2 + ] + + try: + relation.objects.bulk_create(relations, ignore_conflicts=True) + except Exception as e: + logger(f"Error creating ImpactedPackage {relation}: {e!r} \n {traceback_format_exc()}") + return 0 + + compute_package_risk_score_bulk(affected_packages_v2) + + return len(relations) + + +@transaction.atomic +def group_advisories_for_package_with_lock(impact, affected_packages_v2, relation, logger): affected_packages_v2 = sorted( affected_packages_v2, key=lambda p: p.purl, @@ -214,14 +243,7 @@ def bulk_create_with_m2m(purls, impact, relation, logger): logger=logger, ) - risk_score = compute_package_risk_score(pkg) - - logger(f"Computed risk score {risk_score} " f"for package {pkg.purl}") - - pkg.risk_score = risk_score - pkg.save() - - return len(relations) + return relations def impacted_package_qs(cutoff_day=2): From e7f19e8b788abdbed9311522c9a60e3ff6e5c80b Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sat, 30 May 2026 18:38:11 +0530 Subject: [PATCH 23/24] Fix grouping Signed-off-by: Tushar Goel --- vulnerabilities/pipes/advisory.py | 2 -- vulnerabilities/pipes/group_advisories.py | 9 +-------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 286cf75dd..3bfb39d08 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -458,8 +458,6 @@ def group_advisories_for_package_with_lock( group_advisory_for_package( pkg, logger=logger, - current_advisory=advisory_obj, - current_advisory_relation="affecting", ) except Exception as e: logger( diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py index d1266ceea..fa01e669f 100644 --- a/vulnerabilities/pipes/group_advisories.py +++ b/vulnerabilities/pipes/group_advisories.py @@ -64,7 +64,7 @@ def delete_and_save_advisory_set(groups, package, relation=None): def group_advisory_for_package( - package, logger=None, current_advisory=None, current_advisory_relation=None + package, logger=None ): """ Group advisories for a given package and save the advisory sets for the package. @@ -90,13 +90,6 @@ def group_advisory_for_package( "impacted_packages__fixed_by_packages", ) - if current_advisory and current_advisory_relation == "affecting": - if not affecting_advisories.filter(id=current_advisory.id).exists(): - affecting_advisories = list(affecting_advisories) + [current_advisory] - elif current_advisory and current_advisory_relation == "fixing": - if not fixed_by_advisories.filter(id=current_advisory.id).exists(): - fixed_by_advisories = list(fixed_by_advisories) + [current_advisory] - try: affected_groups: List[Group] = merge_advisories(affecting_advisories, package) fixed_by_groups: List[Group] = merge_advisories(fixed_by_advisories, package) From fef52fe6521c15b9e523b5862f8114b1a5b1c5af Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sun, 31 May 2026 00:29:24 +0530 Subject: [PATCH 24/24] Add on impacted packages model Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 8 ++- .../0135_impactedpackage_is_latest.py | 22 +++++++ ...0136_populate_impactedpackage_is_latest.py | 22 +++++++ vulnerabilities/models.py | 62 ++++++++++++------- vulnerabilities/pipes/advisory.py | 3 + vulnerabilities/pipes/export.py | 8 ++- vulnerabilities/pipes/group_advisories.py | 4 +- 7 files changed, 100 insertions(+), 29 deletions(-) create mode 100644 vulnerabilities/migrations/0135_impactedpackage_is_latest.py create mode 100644 vulnerabilities/migrations/0136_populate_impactedpackage_is_latest.py diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index b9e0d94dc..bfc56054d 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -450,11 +450,13 @@ def create(self, request, *args, **kwargs): max_advisories = serializer.validated_data["max_advisories"] if not purls: - impacted = ImpactedPackageAffecting.objects.filter(package_id=OuterRef("id")) + latest_impacts = ImpactedPackageAffecting.objects.filter( + package_id=OuterRef("pk"), + impacted_package__is_latest=True, + ) query = ( - PackageV2.objects.annotate(has_vuln=Exists(impacted)) - .filter(has_vuln=True) + PackageV2.objects.filter(Exists(latest_impacts)) .values_list("package_url", flat=True) .order_by("package_url") ) diff --git a/vulnerabilities/migrations/0135_impactedpackage_is_latest.py b/vulnerabilities/migrations/0135_impactedpackage_is_latest.py new file mode 100644 index 000000000..f930a3c84 --- /dev/null +++ b/vulnerabilities/migrations/0135_impactedpackage_is_latest.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.11 on 2026-05-30 17:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0134_alter_advisoryset_unique_together"), + ] + + operations = [ + migrations.AddField( + model_name="impactedpackage", + name="is_latest", + field=models.BooleanField( + db_index=True, + default=False, + help_text="Indicates whether this is the latest impact for the advisory.", + ), + ), + ] diff --git a/vulnerabilities/migrations/0136_populate_impactedpackage_is_latest.py b/vulnerabilities/migrations/0136_populate_impactedpackage_is_latest.py new file mode 100644 index 000000000..32690f27a --- /dev/null +++ b/vulnerabilities/migrations/0136_populate_impactedpackage_is_latest.py @@ -0,0 +1,22 @@ +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0135_impactedpackage_is_latest"), + ] + + + def mark_latest_impacted_packages(apps, schema_editor): + AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2") + ImpactedPackage = apps.get_model("vulnerabilities", "ImpactedPackage") + + print("\nMarking impacted packages for latest V2 advisories as is_latest=True.") + latest_advisory_ids = AdvisoryV2.objects.filter(is_latest=True).values_list("id", flat=True) + ImpactedPackage.objects.filter(advisory_id__in=latest_advisory_ids).update(is_latest=True) + + + operations = [ + migrations.RunPython(code=mark_latest_impacted_packages, reverse_code=migrations.RunPython.noop), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 1a98f3bc9..36a14521a 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2934,38 +2934,44 @@ def latest_for_avids(self, avids): return self.filter(avid__in=avids).latest_per_avid() def latest_affecting_advisories_for_purl(self, purl): - adv_ids = ImpactedPackageAffecting.objects.filter(package__package_url=purl).values_list( - "impacted_package__advisory_id", - flat=True, - ) + adv_ids = ImpactedPackage.objects.filter( + affecting_packages__package_url=purl, is_latest=True + ).values_list("advisory_id", flat=True) + return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_affecting_advisories_for_purls(self, purls): - adv_ids = ImpactedPackageAffecting.objects.filter( - package__package_url__in=purls + adv_ids = ImpactedPackage.objects.filter( + affecting_packages__package_url__in=purls, is_latest=True ).values_list( - "impacted_package__advisory_id", + "advisory_id", flat=True, ) return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_affecting_advisories_for_packages(self, purls): - adv_ids = ImpactedPackageAffecting.objects.filter(package__in=purls).values_list( - "impacted_package__advisory_id", + adv_ids = ImpactedPackage.objects.filter( + affecting_packages__package_url__in=purls, is_latest=True + ).values_list( + "advisory_id", flat=True, ) return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_fixed_by_advisories_for_purl(self, purl): - adv_ids = ImpactedPackageFixedBy.objects.filter(package__package_url=purl).values_list( - "impacted_package__advisory_id", + adv_ids = ImpactedPackage.objects.filter( + fixed_by_packages__package_url=purl, is_latest=True + ).values_list( + "advisory_id", flat=True, ) return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_fixed_by_advisories_for_purls(self, purls): - adv_ids = ImpactedPackageFixedBy.objects.filter(package__package_url__in=purls).values_list( - "impacted_package__advisory_id", + adv_ids = ImpactedPackage.objects.filter( + fixed_by_packages__package_url__in=purls, is_latest=True + ).values_list( + "advisory_id", flat=True, ) @@ -2973,14 +2979,18 @@ def latest_fixed_by_advisories_for_purls(self, purls): def latest_advisories_for_purls(self, purls): adv_ids = ( - ImpactedPackageAffecting.objects.filter(package__package_url__in=purls) + ImpactedPackage.objects.filter( + affecting_packages__package_url__in=purls, is_latest=True + ) .values_list( - "impacted_package__advisory_id", + "advisory_id", flat=True, ) .union( - ImpactedPackageFixedBy.objects.filter(package__package_url__in=purls).values_list( - "impacted_package__advisory_id", + ImpactedPackage.objects.filter( + fixed_by_packages__package_url__in=purls, is_latest=True + ).values_list( + "advisory_id", flat=True, ) ) @@ -2991,14 +3001,16 @@ def latest_advisories_for_purls(self, purls): def latest_advisories_for_purl(self, purl): adv_ids = ( - ImpactedPackageAffecting.objects.filter(package__package_url=purl) + ImpactedPackage.objects.filter(affecting_packages__package_url=purl, is_latest=True) .values_list( - "impacted_package__advisory_id", + "advisory_id", flat=True, ) .union( - ImpactedPackageFixedBy.objects.filter(package__package_url=purl).values_list( - "impacted_package__advisory_id", + ImpactedPackage.objects.filter( + fixed_by_packages__package_url=purl, is_latest=True + ).values_list( + "advisory_id", flat=True, ) ) @@ -3352,6 +3364,14 @@ class ImpactedPackage(models.Model): help_text="Timestamp of the last successful vers range unfurl.", ) + is_latest = models.BooleanField( + default=False, + blank=False, + null=False, + db_index=True, + help_text="Indicates whether this is the latest impact for the advisory.", + ) + def to_dict(self): from vulnerabilities.utils import purl_to_dict diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 3bfb39d08..3f8b70a0a 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -374,6 +374,8 @@ def insert_advisory_v2( advisory_obj.risk_score = round(risk_score, 1) if risk_score is not None else None advisory_obj.save() + # Set all impacts as is_latest=False for the advisory before creating new impacts with is_latest=True. This ensures that only the impacts related to the latest advisory are marked as latest. + ImpactedPackage.objects.filter(advisory__avid=advisory_obj.avid).update(is_latest=False) for affected_pkg in advisory.affected_packages: impact = ImpactedPackage.objects.create( advisory=advisory_obj, @@ -386,6 +388,7 @@ def insert_advisory_v2( fixed_vers=( str(affected_pkg.fixed_version_range) if affected_pkg.fixed_version_range else None ), + is_latest=True, ) package_affected_purls, package_fixed_purls = get_exact_purls_v2( affected_package=affected_pkg, diff --git a/vulnerabilities/pipes/export.py b/vulnerabilities/pipes/export.py index 8b77d53cb..806bbd07a 100644 --- a/vulnerabilities/pipes/export.py +++ b/vulnerabilities/pipes/export.py @@ -25,7 +25,9 @@ def package_prefetched_qs(checkpoint): .prefetch_related( Prefetch( "affected_in_impacts", - queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related( + queryset=ImpactedPackage.objects.filter(is_latest=True) + .only("advisory_id") + .prefetch_related( Prefetch( "advisory", queryset=AdvisoryV2.objects.only("avid"), @@ -34,7 +36,9 @@ def package_prefetched_qs(checkpoint): ), Prefetch( "fixed_in_impacts", - queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related( + queryset=ImpactedPackage.objects.filter(is_latest=True) + .only("advisory_id") + .prefetch_related( Prefetch( "advisory", queryset=AdvisoryV2.objects.only("avid"), diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py index fa01e669f..74e794c3e 100644 --- a/vulnerabilities/pipes/group_advisories.py +++ b/vulnerabilities/pipes/group_advisories.py @@ -63,9 +63,7 @@ def delete_and_save_advisory_set(groups, package, relation=None): print(f"Successfully saved advisory sets for package: {package.purl}") -def group_advisory_for_package( - package, logger=None -): +def group_advisory_for_package(package, logger=None): """ Group advisories for a given package and save the advisory sets for the package. """