diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f701c0802..9960233e2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,8 @@ next release --------------------- - WARNING: Vulnerablecode V1 API and UI has stopped supporting Ubuntu OVAL advisories, please shift to V3 API for new Ubuntu advisories. +- WARNING: We will deprecate improver pipelines for calculating package version rank, grouping advisories for packages and calculating risk scores in the next release, we are doing it at advisory import time instead of as separate pipelines, this will improve the performance and consistency of the data. +- Calculate package verion rank, group advisories for packages and package risk score and advisory risk score during import of advisories. - Add attribute ``pipeline_id`` to AdvisoryV2 to track the pipeline that created the advisory, also rename existing ``datasource_id`` and AVIDs. Version v38.6.0 diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 634476ddf..e34d9850e 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -8,12 +8,9 @@ # from collections import defaultdict -from typing import List from urllib.parse import urlencode -from django.db.models import Exists from django.db.models import Max -from django.db.models import OuterRef from django.db.models import Prefetch from django_filters import rest_framework as filters from drf_spectacular.utils import extend_schema @@ -23,6 +20,7 @@ from rest_framework.reverse import reverse from rest_framework.throttling import AnonRateThrottle +from vulnerabilities.models import SSVC from vulnerabilities.models import AdvisoryAlias from vulnerabilities.models import AdvisoryReference from vulnerabilities.models import AdvisorySet @@ -30,14 +28,11 @@ from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import AdvisoryWeakness -from vulnerabilities.models import Group from vulnerabilities.models import GroupedAdvisory from vulnerabilities.models import ImpactedPackageAffecting from vulnerabilities.models import PackageV2 from vulnerabilities.throttling import PermissionBasedUserRateThrottle from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS -from vulnerabilities.utils import get_advisories_from_groups -from vulnerabilities.utils import merge_and_save_grouped_advisories class PackageQuerySerializer(serializers.Serializer): @@ -48,6 +43,7 @@ class PackageQuerySerializer(serializers.Serializer): ) details = serializers.BooleanField(default=False) ignore_qualifiers_subpath = serializers.BooleanField(default=False) + max_advisories = serializers.IntegerField(default=100, min_value=1, max_value=10000) def validate(self, data): if not data["purls"]: @@ -227,29 +223,54 @@ def get_affected_by_vulnerabilities(self, package): result = [] for adv in advisories: - fixed = impact_map.get(adv["avid"]) - adv.pop("avid", None) + fixed = impact_map.get(adv["advisory_uid"]) or [] + resource_url = None + + if request := self.context.get("request", None): + resource_url = adv.get("resource_url") or None + resource_url = request.build_absolute_uri(location=resource_url) result.append( { - **adv, + "advisory_id": adv["advisory_id"], + "advisory_uid": adv["advisory_uid"], + "aliases": adv["aliases"], + "summary": adv["summary"], + "weighted_severity": adv["weighted_severity"], + "exploitability": adv["exploitability"], + "risk_score": adv["risk_score"], + "ssvc_trees": adv["ssvc_trees"], "fixed_by_packages": fixed, + "resource_url": resource_url, } ) return result - advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) + if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: + advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + package.package_url + ) - advisories = [] + advisories = [] - if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: advisories_ids = advisories_qs.only("id") advisories_ids = list(advisories_ids[:101]) - if len(advisories_ids) > 100: + if len(advisories_ids) > self.context.get("max_advisories", 100): return None + advisories_qs = advisories_qs.prefetch_related( + "aliases", + Prefetch( + "related_ssvcs", + queryset=SSVC.objects.select_related("source_advisory") + .only("id", "decision", "options", "vector", "source_advisory__url") + .distinct("source_advisory__url"), + to_attr="prefetched_ssvc_trees", + ), + ) + advisory_by_avid = {adv.avid: adv for adv in advisories_qs} avids = advisory_by_avid.keys() @@ -265,8 +286,14 @@ def get_affected_by_vulnerabilities(self, package): for advisory in advisories_qs: impact = impact_by_avid.get(advisory.avid) - if not impact: - continue + fixed_by_packages = [] + if impact: + fixed_by_packages = [pkg.purl for pkg in impact.fixed_by_packages.all()] + + resource_url = None + + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri(location=advisory.get_absolute_url()) result.append( { @@ -274,73 +301,80 @@ def get_affected_by_vulnerabilities(self, package): "advisory_uid": advisory.avid, "aliases": [alias.alias for alias in advisory.aliases.all()], "summary": advisory.summary, - "severity": advisory.weighted_severity, + "weighted_severity": advisory.weighted_severity, "exploitability": advisory.exploitability, "risk_score": advisory.risk_score, - "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], + "fixed_by_packages": fixed_by_packages, + "resource_url": resource_url, + "ssvc_trees": [ + { + "vector": ssvc.vector, + "decision": ssvc.decision, + "options": ssvc.options, + "source_url": ssvc.source_advisory.url, + } + for ssvc in advisory.prefetched_ssvc_trees + ], } ) return result - if not advisories: - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - advisories_qs = advisories_qs.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, advisories_qs, "affecting" - ) - return self.return_advisories_data(package, advisories_qs, advisories) - def get_fixing_vulnerabilities(self, package): advisories = self.context["fixing_advisory_map"].get(package.id, []) - if advisories: - return advisories + results = [] + resource_url = None + for advisory in advisories: + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri(location=advisory["resource_url"]) + results.append( + { + "advisory_id": advisory["advisory_id"], + "resource_url": resource_url, + "advisory_uid": advisory["advisory_uid"], + } + ) + if results: + return results advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) - if not package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: advisories_ids = advisories_qs.only("id") advisories_ids = list(advisories_ids[:101]) - if len(advisories_ids) > 100: + if len(advisories_ids) > self.context.get("max_advisories", 100): return None results = [] for advisory in advisories_qs: + resource_url = None + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri(location=advisory.get_absolute_url()) results.append( { "advisory_id": advisory.advisory_id.split("/")[-1], "advisory_uid": advisory.avid, + "resource_url": resource_url, } ) return results - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - advisories_qs = advisories_qs.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - if not advisories_qs.exists(): - return [] - advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, advisories_qs, "fixing" - ) - return self.return_fixing_advisories_data(advisories) - def return_fixing_advisories_data(self, advisories): result = [] for advisory in advisories: assert isinstance(advisory, GroupedAdvisory) + resource_url = None + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri( + location=advisory.advisory.get_absolute_url() + ) result.append( { "advisory_id": advisory.identifier, "advisory_uid": advisory.advisory.avid, + "resource_url": resource_url, } ) @@ -361,22 +395,28 @@ def return_advisories_data(self, package, advisories_qs, advisories): result = [] for advisory in advisories: assert isinstance(advisory, GroupedAdvisory) + resource_url = None + fixed_by_packages = [] + if request := self.context.get("request", None): + resource_url = request.build_absolute_uri( + location=advisory.advisory.get_absolute_url() + ) impact = impact_by_avid.get(advisory.advisory.avid) - if not impact: - continue + if impact: + fixed_by_packages = list(set([pkg.purl for pkg in impact.fixed_by_packages.all()])) result.append( { "advisory_id": advisory.identifier, "advisory_uid": advisory.advisory.avid, "aliases": [alias.alias for alias in advisory.aliases], + "summary": advisory.advisory.summary, "weighted_severity": advisory.weighted_severity, "exploitability": advisory.exploitability, "risk_score": advisory.risk_score, - "summary": advisory.advisory.summary, - "fixed_by_packages": list( - set([pkg.purl for pkg in impact.fixed_by_packages.all()]) - ), + "fixed_by_packages": fixed_by_packages, + "resource_url": resource_url, + "ssvc_trees": advisory.ssvc_trees, } ) @@ -405,16 +445,10 @@ def create(self, request, *args, **kwargs): purls = serializer.validated_data["purls"] details = serializer.validated_data["details"] ignore_qualifiers_subpath = serializer.validated_data["ignore_qualifiers_subpath"] + max_advisories = serializer.validated_data["max_advisories"] if not purls: - impacted = ImpactedPackageAffecting.objects.filter(package_id=OuterRef("id")) - - query = ( - PackageV2.objects.annotate(has_vuln=Exists(impacted)) - .filter(has_vuln=True) - .values_list("package_url", flat=True) - .order_by("package_url") - ) + query = PackageV2.objects.all_vulnerable_purls().order_by("package_url") page = self.paginate_queryset(query) return self.get_paginated_response(page) @@ -436,13 +470,13 @@ def create(self, request, *args, **kwargs): if not details: if ignore_qualifiers_subpath: query = ( - PackageV2.objects.filter(plain_package_url__in=plain_purls) + PackageV2.objects.filter_plain_purls(plain_purls) .values_list("plain_package_url", flat=True) .order_by("plain_package_url") ) else: query = ( - PackageV2.objects.filter(package_url__in=purls) + PackageV2.objects.filter_purls(purls) .order_by("package_url") .values_list("package_url", flat=True) ) @@ -451,11 +485,9 @@ def create(self, request, *args, **kwargs): return self.get_paginated_response(page) if ignore_qualifiers_subpath: - query = PackageV2.objects.filter(plain_package_url__in=plain_purls).order_by( - "plain_package_url" - ) + query = PackageV2.objects.filter_plain_purls(plain_purls).order_by("plain_package_url") else: - query = PackageV2.objects.filter(package_url__in=purls).order_by("package_url") + query = PackageV2.objects.filter_purls(purls).order_by("package_url") page = self.paginate_queryset(query) affected_advisory_map = get_affected_advisories_bulk(page) @@ -469,6 +501,7 @@ def create(self, request, *args, **kwargs): "advisory_map": affected_advisory_map, "impact_map": impact_map, "fixing_advisory_map": fixing_advisory_map, + "max_advisories": max_advisories, }, ) return self.get_paginated_response(serializer.data) @@ -583,7 +616,27 @@ def get_affected_advisories_bulk(packages): relation_type="affecting", ) .select_related("primary_advisory") - .prefetch_related(Prefetch("aliases", queryset=AdvisoryAlias.objects.only("alias"))) + .prefetch_related( + Prefetch("aliases", queryset=AdvisoryAlias.objects.only("alias")), + Prefetch( + "members", + queryset=AdvisorySetMember.objects.select_related("advisory").prefetch_related( + Prefetch( + "advisory__related_ssvcs", + queryset=SSVC.objects.select_related("source_advisory") + .only( + "id", + "options", + "decision", + "vector", + "source_advisory__url", + ) + .distinct("source_advisory__url"), + to_attr="prefetched_ssvc_trees", + ) + ), + ), + ) .annotate( max_severity=Max( "members__advisory__weighted_severity", @@ -627,6 +680,22 @@ def get_affected_advisories_bulk(packages): identifier = primary.advisory_id.split("/")[-1] aliases = [a for a in adv._aliases_cache if a != identifier] + all_ssvc = [] + + for member in adv.members.all(): + all_ssvc.extend(member.advisory.prefetched_ssvc_trees) + + ssvcs = [] + + for ssvc in all_ssvc: + ssvcs.append( + { + "vector": ssvc.vector, + "decision": ssvc.decision, + "options": ssvc.options, + "source_url": ssvc.source_advisory.url, + } + ) grouped.append( { @@ -637,6 +706,8 @@ def get_affected_advisories_bulk(packages): "exploitability": exploitability, "risk_score": risk_score, "summary": primary.summary, + "resource_url": primary.get_absolute_url(), + "ssvc_trees": ssvcs, } ) @@ -649,7 +720,9 @@ def get_impacts_bulk(packages): package_ids = [p.id for p in packages] impacts = ( - ImpactedPackageAffecting.objects.filter(package_id__in=package_ids) + ImpactedPackageAffecting.objects.filter( + package_id__in=package_ids, impacted_package__is_latest=True + ) .select_related("impacted_package__advisory") .prefetch_related( Prefetch( @@ -697,7 +770,7 @@ def get_fixing_advisories_bulk(packages): package_map = defaultdict(list) for adv in advisory_sets: - package_map[adv.package_id].append(adv.primary_advisory.advisory_id) + package_map[adv.package_id].append(adv.primary_advisory) result = {} @@ -705,9 +778,13 @@ def get_fixing_advisories_bulk(packages): groups = package_map.get(package.id, []) grouped = [] - for adv_id in groups: + for advisory in groups: grouped.append( - {"advisory_id": adv_id.split("/")[-1], "advisory_uid": adv_id.split("/")[-1]} + { + "advisory_id": advisory.advisory_id.split("/")[-1], + "resource_url": advisory.get_absolute_url(), + "advisory_uid": advisory.avid, + } ) result[package.id] = grouped diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 5b47a7cf1..b1db301d4 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -11,7 +11,6 @@ from vulnerabilities.improvers import vulnerability_status from vulnerabilities.pipelines import add_cvss31_to_CVEs from vulnerabilities.pipelines import compute_package_risk -from vulnerabilities.pipelines import compute_package_version_rank from vulnerabilities.pipelines import enhance_with_exploitdb from vulnerabilities.pipelines import enhance_with_kev from vulnerabilities.pipelines import enhance_with_metasploit @@ -32,7 +31,6 @@ enhance_with_metasploit as enhance_with_metasploit_v2, ) from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2 -from vulnerabilities.pipelines.v2_improvers import group_advisories_for_packages from vulnerabilities.pipelines.v2_improvers import reference_collect_commits from vulnerabilities.pipelines.v2_improvers import relate_severities from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2 @@ -61,7 +59,6 @@ enhance_with_metasploit.MetasploitImproverPipeline, enhance_with_exploitdb.ExploitDBImproverPipeline, compute_package_risk.ComputePackageRiskPipeline, - compute_package_version_rank.ComputeVersionRankPipeline, add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline, remove_duplicate_advisories.RemoveDuplicateAdvisoriesPipeline, populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline, @@ -75,7 +72,6 @@ collect_ssvc_trees.CollectSSVCPipeline, relate_severities.RelateSeveritiesPipeline, archive_urls.ArchiveImproverPipeline, - group_advisories_for_packages.GroupAdvisoriesForPackages, compute_advisory_todo_v2.ComputeToDo, reference_collect_commits.CollectReferencesFixCommitsPipeline, enhance_with_github_poc.GithubPocsImproverPipeline, diff --git a/vulnerabilities/migrations/0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more.py b/vulnerabilities/migrations/0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more.py new file mode 100644 index 000000000..8f45487b2 --- /dev/null +++ b/vulnerabilities/migrations/0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 5.2.11 on 2026-05-26 08:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0132_migrate_advisoryv2_datasource_ids"), + ] + + operations = [ + migrations.AlterField( + model_name="advisoryv2", + name="advisory_id", + field=models.CharField( + db_index=True, + help_text="An advisory is a unique vulnerability identifier in some database, such as PYSEC-2020-2233", + max_length=200, + ), + ), + migrations.AlterField( + model_name="advisoryv2", + name="avid", + field=models.CharField( + help_text="Unique ID for the datasource used for this advisory .e.g.: pysec_importer_v2/PYSEC-2020-2233", + max_length=250, + ), + ), + migrations.AlterField( + model_name="advisoryv2", + name="datasource_id", + field=models.CharField( + db_index=True, + help_text="Unique ID for the datasource used for this advisory .e.g.: nginx", + max_length=50, + ), + ), + migrations.AlterField( + model_name="advisoryv2", + name="pipeline_id", + field=models.CharField( + db_index=True, + help_text="Unique ID for the pipeline used for this advisory .e.g.: nginx_importer_v2", + max_length=50, + ), + ), + ] diff --git a/vulnerabilities/migrations/0134_alter_advisoryset_unique_together.py b/vulnerabilities/migrations/0134_alter_advisoryset_unique_together.py new file mode 100644 index 000000000..7b0c2a4be --- /dev/null +++ b/vulnerabilities/migrations/0134_alter_advisoryset_unique_together.py @@ -0,0 +1,17 @@ +# Generated by Django 5.2.11 on 2026-05-28 13:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more"), + ] + + operations = [ + migrations.AlterUniqueTogether( + name="advisoryset", + unique_together={("package", "relation_type", "primary_advisory")}, + ), + ] diff --git a/vulnerabilities/migrations/0135_impactedpackage_is_latest.py b/vulnerabilities/migrations/0135_impactedpackage_is_latest.py new file mode 100644 index 000000000..f930a3c84 --- /dev/null +++ b/vulnerabilities/migrations/0135_impactedpackage_is_latest.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.11 on 2026-05-30 17:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0134_alter_advisoryset_unique_together"), + ] + + operations = [ + migrations.AddField( + model_name="impactedpackage", + name="is_latest", + field=models.BooleanField( + db_index=True, + default=False, + help_text="Indicates whether this is the latest impact for the advisory.", + ), + ), + ] diff --git a/vulnerabilities/migrations/0136_populate_impactedpackage_is_latest.py b/vulnerabilities/migrations/0136_populate_impactedpackage_is_latest.py new file mode 100644 index 000000000..32690f27a --- /dev/null +++ b/vulnerabilities/migrations/0136_populate_impactedpackage_is_latest.py @@ -0,0 +1,22 @@ +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0135_impactedpackage_is_latest"), + ] + + + def mark_latest_impacted_packages(apps, schema_editor): + AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2") + ImpactedPackage = apps.get_model("vulnerabilities", "ImpactedPackage") + + print("\nMarking impacted packages for latest V2 advisories as is_latest=True.") + latest_advisory_ids = AdvisoryV2.objects.filter(is_latest=True).values_list("id", flat=True) + ImpactedPackage.objects.filter(advisory_id__in=latest_advisory_ids).update(is_latest=True) + + + operations = [ + migrations.RunPython(code=mark_latest_impacted_packages, reverse_code=migrations.RunPython.noop), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 69253f54b..3feb8e339 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -19,6 +19,7 @@ from itertools import groupby from operator import attrgetter from traceback import format_exc as traceback_format_exc +from typing import Dict from typing import List from typing import NamedTuple from typing import Optional @@ -2933,38 +2934,44 @@ def latest_for_avids(self, avids): return self.filter(avid__in=avids).latest_per_avid() def latest_affecting_advisories_for_purl(self, purl): - adv_ids = ImpactedPackageAffecting.objects.filter(package__package_url=purl).values_list( - "impacted_package__advisory_id", - flat=True, - ) + adv_ids = ImpactedPackage.objects.filter( + affecting_packages__package_url=purl, is_latest=True + ).values_list("advisory_id", flat=True) + return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_affecting_advisories_for_purls(self, purls): - adv_ids = ImpactedPackageAffecting.objects.filter( - package__package_url__in=purls + adv_ids = ImpactedPackage.objects.filter( + affecting_packages__package_url__in=purls, is_latest=True ).values_list( - "impacted_package__advisory_id", + "advisory_id", flat=True, ) return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_affecting_advisories_for_packages(self, purls): - adv_ids = ImpactedPackageAffecting.objects.filter(package__in=purls).values_list( - "impacted_package__advisory_id", + adv_ids = ImpactedPackage.objects.filter( + affecting_packages__package_url__in=purls, is_latest=True + ).values_list( + "advisory_id", flat=True, ) return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_fixed_by_advisories_for_purl(self, purl): - adv_ids = ImpactedPackageFixedBy.objects.filter(package__package_url=purl).values_list( - "impacted_package__advisory_id", + adv_ids = ImpactedPackage.objects.filter( + fixed_by_packages__package_url=purl, is_latest=True + ).values_list( + "advisory_id", flat=True, ) return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_fixed_by_advisories_for_purls(self, purls): - adv_ids = ImpactedPackageFixedBy.objects.filter(package__package_url__in=purls).values_list( - "impacted_package__advisory_id", + adv_ids = ImpactedPackage.objects.filter( + fixed_by_packages__package_url__in=purls, is_latest=True + ).values_list( + "advisory_id", flat=True, ) @@ -2972,14 +2979,18 @@ def latest_fixed_by_advisories_for_purls(self, purls): def latest_advisories_for_purls(self, purls): adv_ids = ( - ImpactedPackageAffecting.objects.filter(package__package_url__in=purls) + ImpactedPackage.objects.filter( + affecting_packages__package_url__in=purls, is_latest=True + ) .values_list( - "impacted_package__advisory_id", + "advisory_id", flat=True, ) .union( - ImpactedPackageFixedBy.objects.filter(package__package_url__in=purls).values_list( - "impacted_package__advisory_id", + ImpactedPackage.objects.filter( + fixed_by_packages__package_url__in=purls, is_latest=True + ).values_list( + "advisory_id", flat=True, ) ) @@ -2990,14 +3001,16 @@ def latest_advisories_for_purls(self, purls): def latest_advisories_for_purl(self, purl): adv_ids = ( - ImpactedPackageAffecting.objects.filter(package__package_url=purl) + ImpactedPackage.objects.filter(affecting_packages__package_url=purl, is_latest=True) .values_list( - "impacted_package__advisory_id", + "advisory_id", flat=True, ) .union( - ImpactedPackageFixedBy.objects.filter(package__package_url=purl).values_list( - "impacted_package__advisory_id", + ImpactedPackage.objects.filter( + fixed_by_packages__package_url=purl, is_latest=True + ).values_list( + "advisory_id", flat=True, ) ) @@ -3033,6 +3046,9 @@ class AdvisorySet(models.Model): created_at = models.DateTimeField(auto_now_add=True) + class Meta: + unique_together = ("package", "relation_type", "primary_advisory") + class AdvisorySetMember(models.Model): @@ -3054,7 +3070,7 @@ class AdvisoryV2(models.Model): # This is similar to a type or a namespace datasource_id = models.CharField( - max_length=200, + max_length=50, blank=False, null=False, db_index=True, @@ -3062,7 +3078,7 @@ class AdvisoryV2(models.Model): ) pipeline_id = models.CharField( - max_length=200, + max_length=50, blank=False, null=False, db_index=True, @@ -3071,7 +3087,7 @@ class AdvisoryV2(models.Model): # This is similar to a name advisory_id = models.CharField( - max_length=500, + max_length=200, blank=False, null=False, unique=False, @@ -3081,7 +3097,7 @@ class AdvisoryV2(models.Model): ) avid = models.CharField( - max_length=500, + max_length=250, blank=False, null=False, help_text="Unique ID for the datasource used for this advisory ." @@ -3348,6 +3364,14 @@ class ImpactedPackage(models.Model): help_text="Timestamp of the last successful vers range unfurl.", ) + is_latest = models.BooleanField( + default=False, + blank=False, + null=False, + db_index=True, + help_text="Indicates whether this is the latest impact for the advisory.", + ) + def to_dict(self): from vulnerabilities.utils import purl_to_dict @@ -3528,6 +3552,24 @@ def with_is_vulnerable(self): ) ) + def all_vulnerable(self): + latest_impacts = ImpactedPackageAffecting.objects.filter( + package_id=OuterRef("pk"), + impacted_package__is_latest=True, + ) + + query = PackageV2.objects.filter(Exists(latest_impacts)) + return query + + def all_vulnerable_purls(self): + return self.all_vulnerable().values_list("package_url", flat=True) + + def filter_plain_purls(self, plain_purls=[]): + return PackageV2.objects.filter(plain_package_url__in=plain_purls) + + def filter_purls(self, purls=[]): + return PackageV2.objects.filter(package_url__in=purls) + def from_purl(self, purl: Union[PackageURL, str]): """ Return a new Package given a ``purl`` PackageURL object or PURL string. @@ -3872,6 +3914,7 @@ class GroupedAdvisory(NamedTuple): weighted_severity: Optional[float] exploitability: Optional[float] risk_score: Optional[float] + ssvc_trees: List[Dict] class AdvisoryPOC(models.Model): diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index dacf7e6c8..3d3da7c94 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -16,6 +16,7 @@ from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.pipes.risk_score import bulk_update from vulnerabilities.risk import compute_vulnerability_risk_factors @@ -177,15 +178,3 @@ def compute_and_store_package_risk_score(self): logger=self.log, ) self.log(f"Successfully added risk score for {updated:,d} package") - - -def bulk_update(model, items, fields, logger): - item_count = 0 - if items: - try: - model.objects.bulk_update(objs=items, fields=fields) - item_count += len(items) - except Exception as e: - logger(f"Error updating {model.__name__}: {e}") - items.clear() - return item_count diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index ea6fc9185..642b5e3d2 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -7,21 +7,17 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from typing import List - -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import Group from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set +from vulnerabilities.pipes.group_advisories import group_advisory_for_package from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS -from vulnerabilities.utils import merge_advisories class GroupAdvisoriesForPackages(VulnerableCodePipeline): """Group advisories for packages that have multiple importers""" pipeline_id = "group_advisories_for_packages" + run_once = True @classmethod def steps(cls): @@ -33,28 +29,4 @@ def group_advisories_for_packages(self): def group_advisoris_for_packages(logger=None): for package in PackageV2.objects.filter(type__in=TYPES_WITH_MULTIPLE_IMPORTERS).iterator(): - logger(f"Grouping advisories for package {package.purl}") - affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( - purl=package.purl - ).prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( - purl=package.purl - ).prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - try: - affected_groups: List[Group] = merge_advisories(affecting_advisories, package) - fixed_by_groups: List[Group] = merge_advisories(fixed_by_advisories, package) - delete_and_save_advisory_set(affected_groups, package, relation="affecting") - delete_and_save_advisory_set(fixed_by_groups, package, relation="fixing") - except Exception as e: - logger(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") - continue + group_advisory_for_package(package, logger=logger) diff --git a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py index 9d874c635..02259d18e 100644 --- a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py +++ b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py @@ -12,6 +12,7 @@ from traceback import format_exc as traceback_format_exc from aboutcode.pipeline import LoopProgress +from django.db import transaction from django.db.models import F from django.db.models import Q from django.utils import timezone @@ -26,6 +27,9 @@ from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipes.fetchcode_utils import get_versions +from vulnerabilities.pipes.group_advisories import group_advisory_for_package +from vulnerabilities.pipes.risk_score import compute_package_risk_score_bulk +from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS from vulnerabilities.utils import update_purl_version @@ -159,8 +163,11 @@ def get_purl_versions(purl, cached_versions, logger): return cached_versions.get(purl) or [] +@transaction.atomic def bulk_create_with_m2m(purls, impact, relation, logger): - """Bulk create PackageV2 and also bulk populate M2M Impact and Package relationships.""" + """Bulk create PackageV2 and also bulk populate M2M Impact and Package relationships. + This function assumes same base purl is used for all versions in ``purls`` list. + """ if not purls: return 0 @@ -171,17 +178,71 @@ def bulk_create_with_m2m(purls, impact, relation, logger): affected_packages_v2.first().calculate_version_rank + if affected_packages_v2.first().type in TYPES_WITH_MULTIPLE_IMPORTERS: + relations = group_advisories_for_package_with_lock( + impact, affected_packages_v2, relation, logger + ) + + else: + relations = [ + relation(impacted_package=impact, package=package) for package in affected_packages_v2 + ] + + try: + relation.objects.bulk_create(relations, ignore_conflicts=True) + except Exception as e: + logger(f"Error creating ImpactedPackage {relation}: {e!r} \n {traceback_format_exc()}") + return 0 + + compute_package_risk_score_bulk(affected_packages_v2) + + return len(relations) + + +@transaction.atomic +def group_advisories_for_package_with_lock(impact, affected_packages_v2, relation, logger): + affected_packages_v2 = sorted( + affected_packages_v2, + key=lambda p: p.purl, + ) + + locked_packages = [] + + for pkg in affected_packages_v2: + locked_pkg = PackageV2.objects.select_for_update().get(id=pkg.id) + + locked_packages.append(locked_pkg) + + locked_packages[0].calculate_version_rank + relations = [ - relation(impacted_package=impact, package=package) for package in affected_packages_v2 + relation( + impacted_package=impact, + package=package, + ) + for package in locked_packages ] try: - relation.objects.bulk_create(relations, ignore_conflicts=True) + relation.objects.bulk_create( + relations, + ignore_conflicts=True, + ) + except Exception as e: - logger(f"Error creating ImpactedPackage {relation}: {e!r} \n {traceback_format_exc()}") + logger( + f"Error creating ImpactedPackage " f"{relation}: {e!r} \n " f"{traceback_format_exc()}" + ) + return 0 + for pkg in locked_packages: - return len(relations) + group_advisory_for_package( + pkg, + logger=logger, + ) + + return relations def impacted_package_qs(cutoff_day=2): diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 732e2e0ab..3f8b70a0a 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -47,6 +47,9 @@ from vulnerabilities.models import VulnerabilityRelatedReference from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness +from vulnerabilities.pipes.group_advisories import group_advisory_for_package +from vulnerabilities.pipes.risk_score import compute_advisory_risk_score +from vulnerabilities.pipes.risk_score import compute_package_risk_score_bulk from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 @@ -297,6 +300,7 @@ def insert_advisory_v2( ): from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PackageV2 + from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS from vulnerabilities.utils import compute_content_id_v2 advisory_obj = None @@ -362,6 +366,16 @@ def insert_advisory_v2( if values: getattr(advisory_obj, field_name).add(*values) + weighted_severity, exploitability, risk_score = compute_advisory_risk_score(advisory_obj) + advisory_obj.weighted_severity = ( + round(weighted_severity, 1) if weighted_severity is not None else None + ) + advisory_obj.exploitability = round(exploitability, 1) if exploitability is not None else None + advisory_obj.risk_score = round(risk_score, 1) if risk_score is not None else None + advisory_obj.save() + + # Set all impacts as is_latest=False for the advisory before creating new impacts with is_latest=True. This ensures that only the impacts related to the latest advisory are marked as latest. + ImpactedPackage.objects.filter(advisory__avid=advisory_obj.avid).update(is_latest=False) for affected_pkg in advisory.affected_packages: impact = ImpactedPackage.objects.create( advisory=advisory_obj, @@ -374,6 +388,7 @@ def insert_advisory_v2( fixed_vers=( str(affected_pkg.fixed_version_range) if affected_pkg.fixed_version_range else None ), + is_latest=True, ) package_affected_purls, package_fixed_purls = get_exact_purls_v2( affected_package=affected_pkg, @@ -387,8 +402,19 @@ def insert_advisory_v2( purls=package_fixed_purls ) - impact.affecting_packages.add(*affected_packages_v2) - impact.fixed_by_packages.add(*fixed_packages_v2) + if affected_pkg.package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + group_advisories_for_package_with_lock( + advisory_obj, impact, affected_packages_v2, fixed_packages_v2, logger=logger + ) + else: + impact.affecting_packages.add(*affected_packages_v2) + impact.fixed_by_packages.add(*fixed_packages_v2) + + compute_package_risk_score_bulk(affected_packages_v2) + if affected_packages_v2: + affected_packages_v2[0].calculate_version_rank + elif fixed_packages_v2: + fixed_packages_v2[0].calculate_version_rank introduced_commit_v2 = get_or_create_advisory_package_commit_patches( affected_pkg.introduced_by_commit_patches @@ -401,6 +427,48 @@ def insert_advisory_v2( return advisory_obj +@transaction.atomic +def group_advisories_for_package_with_lock( + advisory_obj, impact, affected_packages_v2, fixed_packages_v2, logger=print +): + from vulnerabilities.models import PackageV2 + + all_packages = sorted( + { + *affected_packages_v2, + *fixed_packages_v2, + }, + key=lambda p: p.purl, + ) + + locked_packages = {} + + for pkg in all_packages: + locked_pkg = PackageV2.objects.select_for_update().get(id=pkg.id) + + locked_packages[pkg.id] = locked_pkg + + affected_packages_v2 = [locked_packages[pkg.id] for pkg in affected_packages_v2] + + fixed_packages_v2 = [locked_packages[pkg.id] for pkg in fixed_packages_v2] + + impact.affecting_packages.add(*affected_packages_v2) + impact.fixed_by_packages.add(*fixed_packages_v2) + + for pkg in list(affected_packages_v2) + list(fixed_packages_v2): + logger(f"Grouping advisories for package: {pkg.purl}") + try: + group_advisory_for_package( + pkg, + logger=logger, + ) + except Exception as e: + logger( + f"Failed to group advisories for package {pkg.purl}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + + @transaction.atomic def import_advisory( advisory: Advisory, diff --git a/vulnerabilities/pipes/export.py b/vulnerabilities/pipes/export.py index 8b77d53cb..806bbd07a 100644 --- a/vulnerabilities/pipes/export.py +++ b/vulnerabilities/pipes/export.py @@ -25,7 +25,9 @@ def package_prefetched_qs(checkpoint): .prefetch_related( Prefetch( "affected_in_impacts", - queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related( + queryset=ImpactedPackage.objects.filter(is_latest=True) + .only("advisory_id") + .prefetch_related( Prefetch( "advisory", queryset=AdvisoryV2.objects.only("avid"), @@ -34,7 +36,9 @@ def package_prefetched_qs(checkpoint): ), Prefetch( "fixed_in_impacts", - queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related( + queryset=ImpactedPackage.objects.filter(is_latest=True) + .only("advisory_id") + .prefetch_related( Prefetch( "advisory", queryset=AdvisoryV2.objects.only("avid"), diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py index 983ac3386..74e794c3e 100644 --- a/vulnerabilities/pipes/group_advisories.py +++ b/vulnerabilities/pipes/group_advisories.py @@ -7,46 +7,118 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from collections import defaultdict +from typing import List + from django.db import transaction +from vulnerabilities.models import AdvisorySet +from vulnerabilities.models import AdvisorySetMember +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import Group + @transaction.atomic def delete_and_save_advisory_set(groups, package, relation=None): - from vulnerabilities.models import AdvisorySet - from vulnerabilities.models import AdvisorySetMember - from vulnerabilities.models import Group AdvisorySet.objects.filter(package=package, relation_type=relation).delete() - membership_to_create = [] - for group in groups: + print(f"Grouping advisories for package: {package.purl}") - assert isinstance(group, Group) - advisory_set = AdvisorySet.objects.create( - package=package, - relation_type=relation, - primary_advisory=group.primary, - ) - - advisory_set.aliases.add(*group.aliases) - advisory_set.save() - - membership_to_create.append( - AdvisorySetMember( - advisory_set=advisory_set, - advisory=group.primary, - is_primary=True, + for group in groups: + try: + advisory_set, _ = AdvisorySet.objects.get_or_create( + package=package, + relation_type=relation, + primary_advisory=group.primary, ) - ) - for adv in group.secondaries: + advisory_set.aliases.add(*group.aliases) + advisory_set.save() + membership_to_create.append( AdvisorySetMember( advisory_set=advisory_set, - advisory=adv, - is_primary=False, + advisory=group.primary, + is_primary=True, ) ) + for adv in group.secondaries: + membership_to_create.append( + AdvisorySetMember( + advisory_set=advisory_set, + advisory=adv, + is_primary=False, + ) + ) + except Exception as e: + print( + f"Failed to create advisory set for package {package.purl} with primary advisory {group.primary}: {e!r}" + ) + continue + AdvisorySetMember.objects.bulk_create(membership_to_create) + print(f"Successfully saved advisory sets for package: {package.purl}") + + +def group_advisory_for_package(package, logger=None): + """ + Group advisories for a given package and save the advisory sets for the package. + """ + from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS + + if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: + return + + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + + fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + + try: + affected_groups: List[Group] = merge_advisories(affecting_advisories, package) + fixed_by_groups: List[Group] = merge_advisories(fixed_by_advisories, package) + delete_and_save_advisory_set(affected_groups, package, relation="affecting") + delete_and_save_advisory_set(fixed_by_groups, package, relation="fixing") + logger(f"Successfully rebuilt advisory sets for package {package.purl}") + except Exception as e: + if logger: + logger(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") + return + + +def merge_advisories(advisories, package): + """ + Merge advisories based on their content hash and identifiers. + """ + from vulnerabilities.utils import compute_advisory_content_hash + from vulnerabilities.utils import get_merged_identifier_groups + + advisories = list(advisories) + + content_hash_map = defaultdict(list) + + for adv in advisories: + content_hash = compute_advisory_content_hash(adv, package) + content_hash_map[content_hash].append(adv) + + final_groups: List[Group] = [] + + for group in content_hash_map.values(): + groups = get_merged_identifier_groups(group) + final_groups.extend(groups) + + return final_groups diff --git a/vulnerabilities/pipes/risk_score.py b/vulnerabilities/pipes/risk_score.py new file mode 100644 index 000000000..8a4bf00cf --- /dev/null +++ b/vulnerabilities/pipes/risk_score.py @@ -0,0 +1,133 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from decimal import ROUND_HALF_UP +from decimal import Decimal + +from django.db.models import Max + +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 + + +def quantize_1(value): + if value is None: + return None + + return Decimal(str(value)).quantize( + Decimal("0.1"), + rounding=ROUND_HALF_UP, + ) + + +def compute_package_risk_score(package, current_advisory_risk_score=None): + """Calculate the risk score for a single PackageV2 object.""" + max_risk = ( + AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) + .aggregate(max_risk=Max("risk_score")) + .get("max_risk") + ) + # include current advisory risk score in the calculation if provided and is higher than the max risk score from the database + if current_advisory_risk_score is not None: + max_risk = max(max_risk or 0, current_advisory_risk_score) + if max_risk is None: + return None + return round(float(max_risk), 1) + + +def compute_package_risk_score_bulk(packages): + """Calculate the risk score for a single PackageV2 object.""" + purls = packages.values_list("package_url", flat=True) + advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purls(purls).only( + "id", "risk_score" + ) + qs = ( + PackageV2.objects.filter( + id__in=packages.values_list("id", flat=True), + affected_in_impacts__advisory__risk_score__isnull=False, + affected_in_impacts__advisory__in=advisories, + ) + .distinct() + .annotate(computed_risk=Max("affected_in_impacts__advisory__risk_score")) + .only("id") + ) + + batch = [] + batch_size = 5000 + updated = 0 + + for pkg in qs.iterator(chunk_size=batch_size): + pkg.risk_score = round(float(pkg.computed_risk), 1) + batch.append(pkg) + + if len(batch) >= batch_size: + updated += bulk_update( + model=PackageV2, + items=batch, + fields=["risk_score"], + ) + batch.clear() + + updated += bulk_update( + model=PackageV2, + items=batch, + fields=["risk_score"], + ) + + +def compute_advisory_risk_score(advisory): + """ + Calculate the risk score for a single AdvisoryV2 object. + Returns a tuple of (weighted_severity, exploitability, risk_score). + """ + from vulnerabilities.risk import compute_vulnerability_risk_factors + + weighted_severity = None + exploitability = None + risk_score = None + + references = advisory.references.all() + exploits = advisory.exploits.all() + + severities = list(advisory.severities.all()) + + for rel in advisory.related_advisory_severities.all(): + severities.extend(rel.severities.all()) + + try: + calculated_weighted_severity, calculated_exploitability = ( + compute_vulnerability_risk_factors( + references=references, + severities=severities, + exploits=exploits, + ) + ) + + weighted_severity = calculated_weighted_severity + exploitability = calculated_exploitability + if exploitability and weighted_severity: + risk_score = min(float(exploitability * weighted_severity), 10.0) + risk_score = round(risk_score, 1) + except Exception as e: + risk_score = None + + return quantize_1(weighted_severity), quantize_1(exploitability), quantize_1(risk_score) + + +def bulk_update(model, items, fields, logger=None): + item_count = 0 + if items: + try: + model.objects.bulk_update(objs=items, fields=fields) + item_count += len(items) + except Exception as e: + if logger: + logger(f"Error updating {model.__name__}: {e}") + items.clear() + return item_count diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py index a1bbe0465..1c6de8e4d 100644 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py @@ -293,13 +293,13 @@ def test_todo_conflict_details_partial_curation(self): "package": { "type": "npm", "namespace": "", - "name": "package1", + "name": "package2", "version": "", "qualifiers": "", "subpath": "", }, "affected_version_range": "vers:npm/>=1.0.0|<=2.0.0", - "fixed_version_range": "vers:npm/2.0.1", + "fixed_version_range": None, "introduced_by_commit_patches": [], "fixed_by_commit_patches": [], }, @@ -312,36 +312,36 @@ def test_todo_conflict_details_partial_curation(self): "qualifiers": "", "subpath": "", }, - "affected_version_range": "vers:npm/>=1.0.0|<=2.0.0", + "affected_version_range": "vers:npm/>=3.0.0|<=3.9.0", "fixed_version_range": None, "introduced_by_commit_patches": [], "fixed_by_commit_patches": [], }, { "package": { - "type": "npm", + "type": "pypi", "namespace": "", - "name": "package2", + "name": "package1", "version": "", "qualifiers": "", "subpath": "", }, - "affected_version_range": "vers:npm/>=3.0.0|<=3.9.0", - "fixed_version_range": None, + "affected_version_range": "vers:pypi/>=1.0.0|<=2.0.0", + "fixed_version_range": "vers:pypi/2.0.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [], }, { "package": { - "type": "pypi", + "type": "npm", "namespace": "", "name": "package1", "version": "", "qualifiers": "", "subpath": "", }, - "affected_version_range": "vers:pypi/>=1.0.0|<=2.0.0", - "fixed_version_range": "vers:pypi/2.0.1", + "affected_version_range": "vers:npm/>=1.0.0|<=2.0.0", + "fixed_version_range": "vers:npm/2.0.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [], }, @@ -382,7 +382,6 @@ def test_todo_conflict_details_partial_curation(self): self.assertEqual(1, AdvisoryToDoV2.objects.count()) self.assertEqual("CONFLICTING_FIXED_BY_PACKAGES", todo.issue_type) print(result_partial_curation) - # breakpoint() self.assertDictEqual(expected_partial_curation_advisory, result_partial_curation) def test_todo_conflict_details_partial_curation_unpaired_purl_and_conflicting_affected_and_fixed( diff --git a/vulnerabilities/tests/test_advisory_merge.py b/vulnerabilities/tests/test_advisory_merge.py index 71c214bbb..c4c738483 100644 --- a/vulnerabilities/tests/test_advisory_merge.py +++ b/vulnerabilities/tests/test_advisory_merge.py @@ -18,12 +18,11 @@ from vulnerabilities.models import Group from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PackageV2 +from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set +from vulnerabilities.pipes.group_advisories import merge_advisories from vulnerabilities.utils import compute_advisory_content_hash -from vulnerabilities.utils import delete_and_save_advisory_set from vulnerabilities.utils import get_advisories_from_groups from vulnerabilities.utils import get_merged_identifier_groups -from vulnerabilities.utils import merge_advisories -from vulnerabilities.utils import merge_and_save_grouped_advisories @pytest.mark.django_db @@ -162,27 +161,6 @@ def test_delete_and_save_advisory_set(self): assert any(m.is_primary for m in members) assert any(not m.is_primary for m in members) - def test_merge_and_save_integration(self): - package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") - - adv1 = self.create_advisory("A1", ["1.0"], ["2.0"]) - adv2 = self.create_advisory("A2", ["1.0"], ["2.0"]) - - alias = AdvisoryAlias.objects.create(alias="CVE-1") - - adv1.aliases.add(alias) - adv2.aliases.add(alias) - - result = merge_and_save_grouped_advisories( - package, - [adv1, adv2], - relation="test", - ) - - assert len(result) == 1 - assert AdvisorySet.objects.count() == 1 - assert AdvisorySetMember.objects.count() == 2 - def test_merge_advisories_separates_different_content(self): package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py index cd944a203..714e3bd3c 100644 --- a/vulnerabilities/tests/test_api_v3.py +++ b/vulnerabilities/tests/test_api_v3.py @@ -39,7 +39,7 @@ def setUp(self): self.package = PackageV2.objects.from_purl(purl="pkg:pypi/sample@1.0.0") self.impact = ImpactedPackage.objects.create( - advisory=self.advisory, base_purl="pkg:pypi/sample" + advisory=self.advisory, base_purl="pkg:pypi/sample", is_latest=True ) self.impact.affecting_packages.add(self.package) @@ -67,7 +67,7 @@ def test_packages_post_without_details(self): def test_packages_post_with_details(self): url = reverse("package-v3-list") - with self.assertNumQueries(31): + with self.assertNumQueries(12): response = self.client.post( url, data={ diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py index 420c8c402..b5c4d4997 100644 --- a/vulnerabilities/tests/test_risk.py +++ b/vulnerabilities/tests/test_risk.py @@ -24,7 +24,6 @@ @pytest.fixture -@pytest.mark.django_db def vulnerability(): vul = Vulnerability(vulnerability_id="VCID-Existing") vul.save() @@ -50,7 +49,6 @@ def vulnerability(): @pytest.fixture -@pytest.mark.django_db def exploit(): vul = Vulnerability(vulnerability_id="VCID-Exploit") vul.save() @@ -58,7 +56,6 @@ def exploit(): @pytest.fixture -@pytest.mark.django_db def vulnerability_with_exploit_ref(): vul = Vulnerability(vulnerability_id="VCID-Exploit-Ref") vul.save() @@ -74,7 +71,6 @@ def vulnerability_with_exploit_ref(): @pytest.fixture -@pytest.mark.django_db def high_epss_score(): vul = Vulnerability(vulnerability_id="VCID-HIGH-EPSS") vul.save() @@ -90,7 +86,6 @@ def high_epss_score(): @pytest.fixture -@pytest.mark.django_db def low_epss_score(): vul = Vulnerability(vulnerability_id="VCID-LOW-EPSS") vul.save() diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 2e618a920..dc72ad4d2 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -35,6 +35,7 @@ import urllib3 from cwe2.database import Database from cwe2.database import InvalidCWEError +from django.db.models import Prefetch from packageurl import PackageURL from packageurl.contrib.django.utils import without_empty_values from univers.version_range import RANGE_CLASS_BY_SCHEMES @@ -43,7 +44,6 @@ from univers.version_range import VersionRange from aboutcode.hashid import build_vcid -from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set logger = logging.getLogger(__name__) @@ -868,29 +868,6 @@ def compute_patch_checksum(patch_text: str): return hashlib.sha512(patch_text.encode("utf-8")).hexdigest() -def merge_advisories(advisories, package): - """ - Merge advisories based on their content hash and identifiers. - """ - from vulnerabilities.models import Group - - advisories = list(advisories) - - content_hash_map = defaultdict(list) - - for adv in advisories: - content_hash = compute_advisory_content_hash(adv, package) - content_hash_map[content_hash].append(adv) - - final_groups: List[Group] = [] - - for group in content_hash_map.values(): - groups = get_merged_identifier_groups(group) - final_groups.extend(groups) - - return final_groups - - def compute_advisory_content_hash(adv, package): """Compute a content hash for an advisory based on its affected and fixed packages for a given package. This is used to determine if two advisories are the same based on their content.""" @@ -906,8 +883,12 @@ def compute_advisory_content_hash(adv, package): ) for impact in adv.impacted_packages.filter(base_purl=str(version_less_purl)): - affected.extend([pkg.package_url for pkg in impact.affecting_packages.all()]) - fixed.extend([pkg.package_url for pkg in impact.fixed_by_packages.all()]) + for pkg in impact.affecting_packages.all(): + if pkg.package_url: + affected.extend([pkg.package_url]) + for pkg in impact.fixed_by_packages.all(): + if pkg.package_url: + fixed.extend([pkg.package_url]) normalized_data = { "affected_packages": normalize_list(affected), @@ -979,10 +960,12 @@ def get_merged_identifier_groups(advisories): return final_groups -def get_advisories_from_groups(groups): +def get_advisories_from_groups(groups, include_ssvc_trees=False): """ Return a list of advisories from the merged groups of advisories. """ + from vulnerabilities.models import SSVC + from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import Group from vulnerabilities.models import GroupedAdvisory @@ -1016,6 +999,35 @@ def get_advisories_from_groups(groups): identifier = group.primary.advisory_id.split("/")[-1] filtered_aliases = [alias for alias in group.aliases if alias.alias != identifier] + ssvc_trees = [] + + if include_ssvc_trees: + + all_advs = [group.primary] + list(group.secondaries) + + advisories_qs = AdvisoryV2.objects.filter( + id__in=[adv.id for adv in all_advs] + ).prefetch_related( + Prefetch( + "related_ssvcs", + queryset=SSVC.objects.select_related("source_advisory") + .only("id", "vector", "decision", "options", "source_advisory__url") + .distinct(), + to_attr="ssvc_trees", + ) + ) + + ssvc_trees = [ + { + "vector": ssvc.vector, + "decision": ssvc.decision, + "options": ssvc.options, + "url": ssvc.source_advisory.url if ssvc.source_advisory else None, + } + for adv in advisories_qs + for ssvc in adv.ssvc_trees + ] + advisories.append( GroupedAdvisory( aliases=filtered_aliases, @@ -1024,23 +1036,13 @@ def get_advisories_from_groups(groups): weighted_severity=weighted_severity, exploitability=exploitability, risk_score=risk_score, + ssvc_trees=ssvc_trees or [], ) ) return advisories -def merge_and_save_grouped_advisories(package, advisories, relation): - """ - Merge advisories based on their content and identifiers and save the merged advisories to the database. - """ - groups = merge_advisories(advisories, package) - delete_and_save_advisory_set(groups, package, relation) - advisories = get_advisories_from_groups(groups) - - return advisories - - TYPES_WITH_MULTIPLE_IMPORTERS = [ "pypi", "maven", diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 3aff06768..76ca438d3 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -53,7 +53,6 @@ from vulnerabilities.throttling import AnonUserUIThrottle from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS from vulnerabilities.utils import get_advisories_from_groups -from vulnerabilities.utils import merge_and_save_grouped_advisories from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerablecode.settings import env @@ -390,42 +389,6 @@ def get_context_data(self, **kwargs): return context - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( - purl=package.purl - ) - - fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( - purl=package.purl - ) - fixed_pkg_details = get_fixed_package_details(package) - context["fixed_package_details"] = fixed_pkg_details - context["grouped"] = True - - affecting_advisories = affecting_advisories.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - affected_by_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, affecting_advisories, "affecting" - ) - - fixed_by_advisories = fixed_by_advisories.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - - fixing_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, fixed_by_advisories, "fixing" - ) - - context["affected_by_advisories_v2"] = affected_by_advisories - context["fixing_advisories_v2"] = fixing_advisories - return context - def get_object(self, queryset=None): if queryset is None: queryset = self.get_queryset() @@ -458,7 +421,9 @@ def get_fixed_package_details(package): p.id: p for p in models.PackageV2.objects.filter(id__in=pkg_ids, is_ghost=False).annotate( is_vulnerable=Exists( - models.ImpactedPackage.objects.filter(affecting_packages=OuterRef("pk")) + models.ImpactedPackage.objects.filter( + affecting_packages=OuterRef("pk"), is_latest=True + ) ) ) } @@ -927,7 +892,7 @@ def get_queryset(self): .prefetch_related( Prefetch( "impacted_packages", - queryset=models.ImpactedPackage.objects.prefetch_related( + queryset=models.ImpactedPackage.objects.filter(is_latest=True).prefetch_related( Prefetch( "affecting_packages", queryset=models.PackageV2.objects.only(