From 530cb52e124c6b1e9765d269d7b233c94bb2f42f Mon Sep 17 00:00:00 2001 From: ziadhany Date: Sat, 9 Nov 2024 02:28:00 +0200 Subject: [PATCH 1/8] Add exploitability and weighted_severity fields to the Vulnerability model. Create a pipeline for vulnerability risk assessment. Signed-off-by: ziad hany --- vulnerabilities/api.py | 3 + ...7_vulnerability_exploitability_and_more.py | 33 +++++++++++ vulnerabilities/models.py | 29 ++++++++++ .../pipelines/compute_package_risk.py | 56 ++++++++++++++++++- vulnerabilities/risk.py | 30 +++++++--- .../templates/vulnerability_details.html | 32 +++++++++++ .../pipelines/test_compute_package_risk.py | 3 +- vulnerabilities/tests/test_api.py | 6 ++ vulnerabilities/tests/test_risk.py | 3 +- 9 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 vulnerabilities/migrations/0077_vulnerability_exploitability_and_more.py diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index b8bb703a6..cb8e68976 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -251,6 +251,9 @@ class Meta: "weaknesses", "exploits", "severity_range_score", + "exploitability", + "weighted_severity", + "risk_score", ] diff --git a/vulnerabilities/migrations/0077_vulnerability_exploitability_and_more.py b/vulnerabilities/migrations/0077_vulnerability_exploitability_and_more.py new file mode 100644 index 000000000..fccaafa0b --- /dev/null +++ b/vulnerabilities/migrations/0077_vulnerability_exploitability_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.16 on 2024-11-08 14:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0076_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="vulnerability", + name="exploitability", + field=models.DecimalField( + decimal_places=2, + help_text="Exploitability refers to the potential or probability of a software package vulnerability being \n exploited by malicious actors to compromise systems, applications, or networks. \n It is determined automatically by the discovery of exploits.", + max_digits=4, + null=True, + ), + ), + migrations.AddField( + model_name="vulnerability", + name="weighted_severity", + field=models.DecimalField( + decimal_places=2, + help_text="Weighted Severity is the maximum value obtained when each Severity is multiplied by its associated Weight/10.", + max_digits=4, + null=True, + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index b95a07297..9eb466d71 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -202,6 +202,35 @@ class Vulnerability(models.Model): choices=VulnerabilityStatusType.choices, default=VulnerabilityStatusType.PUBLISHED ) + exploitability = models.DecimalField( + null=True, + max_digits=4, + decimal_places=2, + help_text="""Exploitability refers to the potential or probability of a software package vulnerability being + exploited by malicious actors to compromise systems, applications, or networks. + It is determined automatically by the discovery of exploits.""", + ) + + weighted_severity = models.DecimalField( + null=True, + max_digits=4, + decimal_places=2, + help_text="Weighted Severity is the maximum value obtained when each Severity is multiplied by its associated Weight/10.", + ) + + @property + def risk_score(self): + """ + Risk expressed as a number ranging from 0 to 10. + Risk is calculated from weighted severity and exploitability values. + It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 + + Risk = min(weighted severity * exploitability, 10) + """ + if self.exploitability is not None and self.weighted_severity is not None: + return f"{min(float(self.exploitability) * float(self.weighted_severity), 10.0):.2f}" + return None + objects = VulnerabilityQuerySet.as_manager() class Meta: diff --git a/vulnerabilities/pipelines/compute_package_risk.py b/vulnerabilities/pipelines/compute_package_risk.py index e5b48ea0e..7ac179689 100644 --- a/vulnerabilities/pipelines/compute_package_risk.py +++ b/vulnerabilities/pipelines/compute_package_risk.py @@ -9,9 +9,12 @@ from aboutcode.pipeline import LoopProgress +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.risk import compute_package_risk +from vulnerabilities.risk import compute_vulnerability_risk class ComputePackageRiskPipeline(VulnerableCodePipeline): @@ -26,7 +29,44 @@ class ComputePackageRiskPipeline(VulnerableCodePipeline): @classmethod def steps(cls): - return (cls.add_package_risk_score,) + return (cls.add_vulnerability_risk_score, cls.add_package_risk_score) + + def add_vulnerability_risk_score(self): + affected_vulnerabilities = Vulnerability.objects.filter( + affectedbypackagerelatedvulnerability__isnull=False + ) + + self.log( + f"Calculating risk for {affected_vulnerabilities.count():,d} vulnerability with a affected packages records" + ) + + progress = LoopProgress(total_iterations=affected_vulnerabilities.count(), logger=self.log) + + updatables = [] + updated_vulnerability_count = 0 + batch_size = 5000 + + for vulnerability in progress.iter(affected_vulnerabilities.paginated()): + + vulnerability = compute_vulnerability_risk(vulnerability) + + if not vulnerability: + continue + + updatables.append(vulnerability) + + if len(updatables) >= batch_size: + updated_vulnerability_count += bulk_update_vulnerability_risk_score( + vulnerabilities=updatables, + logger=self.log, + ) + updated_vulnerability_count += bulk_update_vulnerability_risk_score( + vulnerabilities=updatables, + logger=self.log, + ) + self.log( + f"Successfully added risk score for {updated_vulnerability_count:,d} vulnerability" + ) def add_package_risk_score(self): affected_packages = Package.objects.filter( @@ -72,3 +112,17 @@ def bulk_update_package_risk_score(packages, logger): logger(f"Error updating packages: {e}") packages.clear() return package_count + + +def bulk_update_vulnerability_risk_score(vulnerabilities, logger): + vulnerabilities_count = 0 + if vulnerabilities: + try: + Vulnerability.objects.bulk_update( + objs=vulnerabilities, fields=["weighted_severity", "exploitability"] + ) + vulnerabilities_count += len(vulnerabilities) + except Exception as e: + logger(f"Error updating vulnerability: {e}") + vulnerabilities.clear() + return vulnerabilities_count diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index 9eb4ac6ec..bc06e898c 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -92,19 +92,31 @@ def get_exploitability_level(exploits, references, severities): def compute_vulnerability_risk(vulnerability: Vulnerability): """ - Risk may be expressed as a number ranging from 0 to 10. - Risk is calculated from weighted severity and exploitability values. - It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 + Computes the risk score for a given vulnerability. - Risk = min(weighted severity * exploitability, 10) + Risk is expressed as a number ranging from 0 to 10 and is calculated based on: + - Weighted severity: a value derived from the associated severities of the vulnerability. + - Exploitability: a measure of how easily the vulnerability can be exploited. + + The risk score is computed as: + Risk = min(weighted_severity * exploitability, 10) + + Args: + vulnerability (Vulnerability): The vulnerability object to compute the risk for. + + Returns: + Vulnerability: The updated vulnerability object with computed risk-related attributes. + + Notes: + - If there are no associated references, severities, or exploits, the computation is skipped. """ references = vulnerability.references severities = vulnerability.severities.select_related("reference") exploits = Exploit.objects.filter(vulnerability=vulnerability) if references.exists() or severities.exists() or exploits.exists(): - weighted_severity = get_weighted_severity(severities) - exploitability = get_exploitability_level(exploits, references, severities) - return min(weighted_severity * exploitability, 10) + vulnerability.weighted_severity = get_weighted_severity(severities) + vulnerability.exploitability = get_exploitability_level(exploits, references, severities) + return vulnerability def compute_package_risk(package: Package): @@ -117,8 +129,8 @@ def compute_package_risk(package: Package): for pkg_related_vul in AffectedByPackageRelatedVulnerability.objects.filter( package=package ).prefetch_related("vulnerability"): - if risk := compute_vulnerability_risk(pkg_related_vul.vulnerability): - result.append(risk) + if risk := pkg_related_vul.vulnerability.risk_score: + result.append(float(risk)) if not result: return diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index d1f2fb6de..ed9f38a7a 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -121,6 +121,38 @@ Status {{ status }} + + + + Exploitability + + {{ vulnerability.exploitability }} + + + + + Weighted Severity + + {{ vulnerability.weighted_severity }} + + + + + Risk + + {{ vulnerability.risk_score }} + + + diff --git a/vulnerabilities/tests/pipelines/test_compute_package_risk.py b/vulnerabilities/tests/pipelines/test_compute_package_risk.py index 7c197e812..1e828d58f 100644 --- a/vulnerabilities/tests/pipelines/test_compute_package_risk.py +++ b/vulnerabilities/tests/pipelines/test_compute_package_risk.py @@ -6,6 +6,7 @@ # See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +from decimal import Decimal import pytest @@ -30,4 +31,4 @@ def test_simple_risk_pipeline(vulnerability): improver.execute() pkg = Package.objects.get(type="pypi", name="foo", version="2.3.0") - assert str(pkg.risk_score) == str(3.11) + assert f"{pkg.risk_score:.2f}" == "3.10" diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 926d3c219..c21451e78 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -300,6 +300,9 @@ def test_api_with_single_vulnerability(self): }, ], "exploits": [], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, } def test_api_with_single_vulnerability_with_filters(self): @@ -346,6 +349,9 @@ def test_api_with_single_vulnerability_with_filters(self): }, ], "exploits": [], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, } diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py index 96d9f9445..5b0aa4c5f 100644 --- a/vulnerabilities/tests/test_risk.py +++ b/vulnerabilities/tests/test_risk.py @@ -170,4 +170,5 @@ def test_get_weighted_severity(vulnerability): @pytest.mark.django_db def test_compute_vulnerability_risk(vulnerability): - assert compute_vulnerability_risk(vulnerability) == 3.1050000000000004 + vulnerability = compute_vulnerability_risk(vulnerability) + assert vulnerability.risk_score == str(3.11) From 86f69275735b68540293bd3c48ec2f512d15853c Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 12 Nov 2024 17:50:09 +0200 Subject: [PATCH 2/8] Optimize performance, refactor, and rename the add_vulnerability_risk_score function. Rename the help text for the model. Signed-off-by: ziad hany --- ...7_vulnerability_exploitability_and_more.py | 8 ++-- vulnerabilities/models.py | 12 ++--- .../pipelines/compute_package_risk.py | 28 ++++++----- vulnerabilities/risk.py | 44 +++++++++--------- .../templates/vulnerability_details.html | 8 ++-- .../pipelines/test_compute_package_risk.py | 2 +- vulnerabilities/tests/test_risk.py | 46 +++++++++++++++++-- 7 files changed, 96 insertions(+), 52 deletions(-) diff --git a/vulnerabilities/migrations/0077_vulnerability_exploitability_and_more.py b/vulnerabilities/migrations/0077_vulnerability_exploitability_and_more.py index fccaafa0b..3a10bf4b3 100644 --- a/vulnerabilities/migrations/0077_vulnerability_exploitability_and_more.py +++ b/vulnerabilities/migrations/0077_vulnerability_exploitability_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.16 on 2024-11-08 14:07 +# Generated by Django 4.2.16 on 2024-11-12 12:16 from django.db import migrations, models @@ -15,7 +15,9 @@ class Migration(migrations.Migration): name="exploitability", field=models.DecimalField( decimal_places=2, - help_text="Exploitability refers to the potential or probability of a software package vulnerability being \n exploited by malicious actors to compromise systems, applications, or networks. \n It is determined automatically by the discovery of exploits.", + help_text="""Exploitability indicates the likelihood that a vulnerability in a software package could + be used by malicious actors to compromise systems, applications, or networks. + This metric is determined automatically based on the discovery of known exploits.""", max_digits=4, null=True, ), @@ -25,7 +27,7 @@ class Migration(migrations.Migration): name="weighted_severity", field=models.DecimalField( decimal_places=2, - help_text="Weighted Severity is the maximum value obtained when each Severity is multiplied by its associated Weight/10.", + help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10.", max_digits=4, null=True, ), diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 9eb466d71..689a7c992 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -206,16 +206,16 @@ class Vulnerability(models.Model): null=True, max_digits=4, decimal_places=2, - help_text="""Exploitability refers to the potential or probability of a software package vulnerability being - exploited by malicious actors to compromise systems, applications, or networks. - It is determined automatically by the discovery of exploits.""", + help_text=""""Exploitability indicates the likelihood that a vulnerability in a software package could be used + by malicious actors to compromise systems, applications, or networks. + This metric is determined automatically based on the discovery of known exploits.""", ) weighted_severity = models.DecimalField( null=True, max_digits=4, decimal_places=2, - help_text="Weighted Severity is the maximum value obtained when each Severity is multiplied by its associated Weight/10.", + help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10.", ) @property @@ -228,8 +228,8 @@ def risk_score(self): Risk = min(weighted severity * exploitability, 10) """ if self.exploitability is not None and self.weighted_severity is not None: - return f"{min(float(self.exploitability) * float(self.weighted_severity), 10.0):.2f}" - return None + risk_score = min(float(self.exploitability) * float(self.weighted_severity), 10.0) + return f"{risk_score:.2f}".rstrip("0").rstrip(".") objects = VulnerabilityQuerySet.as_manager() diff --git a/vulnerabilities/pipelines/compute_package_risk.py b/vulnerabilities/pipelines/compute_package_risk.py index 7ac179689..57409a22e 100644 --- a/vulnerabilities/pipelines/compute_package_risk.py +++ b/vulnerabilities/pipelines/compute_package_risk.py @@ -9,12 +9,11 @@ from aboutcode.pipeline import LoopProgress -from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.risk import compute_package_risk -from vulnerabilities.risk import compute_vulnerability_risk +from vulnerabilities.risk import compute_vulnerability_risk_factors class ComputePackageRiskPipeline(VulnerableCodePipeline): @@ -29,11 +28,16 @@ class ComputePackageRiskPipeline(VulnerableCodePipeline): @classmethod def steps(cls): - return (cls.add_vulnerability_risk_score, cls.add_package_risk_score) + return ( + cls.compute_and_store_vulnerability_risk_score, + cls.compute_and_store_package_risk_score, + ) - def add_vulnerability_risk_score(self): - affected_vulnerabilities = Vulnerability.objects.filter( - affectedbypackagerelatedvulnerability__isnull=False + def compute_and_store_vulnerability_risk_score(self): + affected_vulnerabilities = ( + Vulnerability.objects.filter(affectedbypackagerelatedvulnerability__isnull=False) + .prefetch_related("references") + .only("references", "exploits") ) self.log( @@ -47,11 +51,13 @@ def add_vulnerability_risk_score(self): batch_size = 5000 for vulnerability in progress.iter(affected_vulnerabilities.paginated()): + references = vulnerability.references + severities = vulnerability.severities.select_related("reference") - vulnerability = compute_vulnerability_risk(vulnerability) - - if not vulnerability: - continue + ( + vulnerability.weighted_severity, + vulnerability.exploitability, + ) = compute_vulnerability_risk_factors(references, severities, vulnerability.exploits) updatables.append(vulnerability) @@ -68,7 +74,7 @@ def add_vulnerability_risk_score(self): f"Successfully added risk score for {updated_vulnerability_count:,d} vulnerability" ) - def add_package_risk_score(self): + def compute_and_store_package_risk_score(self): affected_packages = Package.objects.filter( affected_by_vulnerabilities__isnull=False ).distinct() diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index bc06e898c..d733f12c6 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -10,6 +10,8 @@ from urllib.parse import urlparse +from django.db.models import Prefetch + from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Exploit from vulnerabilities.models import Package @@ -27,6 +29,8 @@ def get_weighted_severity(severities): by its associated Weight/10. Example of Weighted Severity: max(7*(10/10), 8*(3/10), 6*(8/10)) = 7 """ + if not severities: + return 0 score_map = { "low": 3, @@ -90,33 +94,21 @@ def get_exploitability_level(exploits, references, severities): return exploit_level -def compute_vulnerability_risk(vulnerability: Vulnerability): +def compute_vulnerability_risk_factors(references, severities, exploits): """ - Computes the risk score for a given vulnerability. - - Risk is expressed as a number ranging from 0 to 10 and is calculated based on: - - Weighted severity: a value derived from the associated severities of the vulnerability. - - Exploitability: a measure of how easily the vulnerability can be exploited. - - The risk score is computed as: - Risk = min(weighted_severity * exploitability, 10) + Compute weighted severity and exploitability for a vulnerability. Args: - vulnerability (Vulnerability): The vulnerability object to compute the risk for. + references (list): References linked to the vulnerability. + severities (list): Severity levels of the vulnerability. + exploits (list): Exploit details for the vulnerability. Returns: - Vulnerability: The updated vulnerability object with computed risk-related attributes. - - Notes: - - If there are no associated references, severities, or exploits, the computation is skipped. + tuple: (weighted_severity, exploitability). """ - references = vulnerability.references - severities = vulnerability.severities.select_related("reference") - exploits = Exploit.objects.filter(vulnerability=vulnerability) - if references.exists() or severities.exists() or exploits.exists(): - vulnerability.weighted_severity = get_weighted_severity(severities) - vulnerability.exploitability = get_exploitability_level(exploits, references, severities) - return vulnerability + weighted_severity = get_weighted_severity(severities) + exploitability = get_exploitability_level(exploits, references, severities) + return weighted_severity, exploitability def compute_package_risk(package: Package): @@ -126,9 +118,15 @@ def compute_package_risk(package: Package): """ result = [] - for pkg_related_vul in AffectedByPackageRelatedVulnerability.objects.filter( + affected_pkg_related_vul = AffectedByPackageRelatedVulnerability.objects.filter( package=package - ).prefetch_related("vulnerability"): + ).prefetch_related( + Prefetch( + "vulnerability", + queryset=Vulnerability.objects.only("weighted_severity", "exploitability"), + ) + ) + for pkg_related_vul in affected_pkg_related_vul: if risk := pkg_related_vul.vulnerability.risk_score: result.append(float(risk)) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index ed9f38a7a..03dd61749 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -124,9 +124,9 @@ + data-tooltip="Exploitability indicates the likelihood that a vulnerability in a software package could be used + by malicious actors to compromise systems, applications, or networks. + This metric is determined automatically based on the discovery of known exploits."> Exploitability {{ vulnerability.exploitability }} @@ -135,7 +135,7 @@ Weighted Severity {{ vulnerability.weighted_severity }} diff --git a/vulnerabilities/tests/pipelines/test_compute_package_risk.py b/vulnerabilities/tests/pipelines/test_compute_package_risk.py index 1e828d58f..b8608cbc0 100644 --- a/vulnerabilities/tests/pipelines/test_compute_package_risk.py +++ b/vulnerabilities/tests/pipelines/test_compute_package_risk.py @@ -31,4 +31,4 @@ def test_simple_risk_pipeline(vulnerability): improver.execute() pkg = Package.objects.get(type="pypi", name="foo", version="2.3.0") - assert f"{pkg.risk_score:.2f}" == "3.10" + assert pkg.risk_score == Decimal("10") diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py index 5b0aa4c5f..8155a11e8 100644 --- a/vulnerabilities/tests/test_risk.py +++ b/vulnerabilities/tests/test_risk.py @@ -15,7 +15,7 @@ from vulnerabilities.models import VulnerabilityRelatedReference from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness -from vulnerabilities.risk import compute_vulnerability_risk +from vulnerabilities.risk import compute_vulnerability_risk_factors from vulnerabilities.risk import get_exploitability_level from vulnerabilities.risk import get_weighted_severity from vulnerabilities.severity_systems import CVSSV3 @@ -169,6 +169,44 @@ def test_get_weighted_severity(vulnerability): @pytest.mark.django_db -def test_compute_vulnerability_risk(vulnerability): - vulnerability = compute_vulnerability_risk(vulnerability) - assert vulnerability.risk_score == str(3.11) +def test_compute_vulnerability_risk_factors(vulnerability): + assert compute_vulnerability_risk_factors( + vulnerability.references, vulnerability.severities, vulnerability.exploits + ) == (6.210000000000001, 2) + assert compute_vulnerability_risk_factors( + vulnerability.references, vulnerability.severities, None + ) == ( + 6.210000000000001, + 0.5, + ) + assert compute_vulnerability_risk_factors( + vulnerability.references, None, vulnerability.exploits + ) == ( + 0, + 2, + ) + assert compute_vulnerability_risk_factors(None, None, None) == (0, 0.5) + + +@pytest.mark.django_db +def test_get_vulnerability_risk_score(vulnerability): + vulnerability.weighted_severity = 6.0 + vulnerability.exploitability = 2 + + assert vulnerability.risk_score == "10" # max risk_score can be reached + + vulnerability.weighted_severity = 6 + vulnerability.exploitability = 0.5 + assert vulnerability.risk_score == "3" + + vulnerability.weighted_severity = 5.6 + vulnerability.exploitability = 0.5 + assert vulnerability.risk_score == "2.8" + + vulnerability.weighted_severity = None + vulnerability.exploitability = 0.5 + assert vulnerability.risk_score is None + + vulnerability.weighted_severity = None + vulnerability.exploitability = None + assert vulnerability.risk_score is None From 405bf864f2424ca82cf8c1444005241ad189c226 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 12 Nov 2024 18:13:59 +0200 Subject: [PATCH 3/8] Resolve migration conflict Signed-off-by: ziad hany --- ..._more.py => 0078_vulnerability_exploitability_and_more.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename vulnerabilities/migrations/{0077_vulnerability_exploitability_and_more.py => 0078_vulnerability_exploitability_and_more.py} (91%) diff --git a/vulnerabilities/migrations/0077_vulnerability_exploitability_and_more.py b/vulnerabilities/migrations/0078_vulnerability_exploitability_and_more.py similarity index 91% rename from vulnerabilities/migrations/0077_vulnerability_exploitability_and_more.py rename to vulnerabilities/migrations/0078_vulnerability_exploitability_and_more.py index 3a10bf4b3..06311036d 100644 --- a/vulnerabilities/migrations/0077_vulnerability_exploitability_and_more.py +++ b/vulnerabilities/migrations/0078_vulnerability_exploitability_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.16 on 2024-11-12 12:16 +# Generated by Django 4.2.16 on 2024-11-12 16:03 from django.db import migrations, models @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ("vulnerabilities", "0076_alter_packagechangelog_software_version_and_more"), + ("vulnerabilities", "0077_alter_packagechangelog_software_version_and_more"), ] operations = [ From ec5b972a825f787de83964c0ecd36b8a2d55d6a0 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Sat, 16 Nov 2024 22:54:25 +0200 Subject: [PATCH 4/8] Resolve migration conflict & add weighted_severity, exploitability to api_v2 Signed-off-by: ziad hany --- vulnerabilities/api_v2.py | 6 +++ ...8_vulnerability_exploitability_and_more.py | 35 --------------- ...2_vulnerability_exploitability_and_more.py | 43 +++++++++++++++++++ vulnerabilities/models.py | 32 +++++++++++++- .../pipelines/compute_package_risk.py | 26 ++++------- vulnerabilities/risk.py | 20 ++++----- vulnerabilities/tests/test_risk.py | 30 ++++++------- 7 files changed, 109 insertions(+), 83 deletions(-) delete mode 100644 vulnerabilities/migrations/0078_vulnerability_exploitability_and_more.py create mode 100644 vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py index b0a3fa125..58771c916 100644 --- a/vulnerabilities/api_v2.py +++ b/vulnerabilities/api_v2.py @@ -67,6 +67,9 @@ class VulnerabilityV2Serializer(serializers.ModelSerializer): weaknesses = WeaknessV2Serializer(many=True) references = VulnerabilityReferenceV2Serializer(many=True, source="vulnerabilityreference_set") severities = VulnerabilitySeverityV2Serializer(many=True) + exploitability = serializers.FloatField(read_only=True) + weighted_severity = serializers.FloatField(read_only=True) + risk_score = serializers.FloatField(read_only=True) class Meta: model = Vulnerability @@ -77,6 +80,9 @@ class Meta: "severities", "weaknesses", "references", + "exploitability", + "weighted_severity", + "risk_score", ] def get_aliases(self, obj): diff --git a/vulnerabilities/migrations/0078_vulnerability_exploitability_and_more.py b/vulnerabilities/migrations/0078_vulnerability_exploitability_and_more.py deleted file mode 100644 index 06311036d..000000000 --- a/vulnerabilities/migrations/0078_vulnerability_exploitability_and_more.py +++ /dev/null @@ -1,35 +0,0 @@ -# Generated by Django 4.2.16 on 2024-11-12 16:03 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("vulnerabilities", "0077_alter_packagechangelog_software_version_and_more"), - ] - - operations = [ - migrations.AddField( - model_name="vulnerability", - name="exploitability", - field=models.DecimalField( - decimal_places=2, - help_text="""Exploitability indicates the likelihood that a vulnerability in a software package could - be used by malicious actors to compromise systems, applications, or networks. - This metric is determined automatically based on the discovery of known exploits.""", - max_digits=4, - null=True, - ), - ), - migrations.AddField( - model_name="vulnerability", - name="weighted_severity", - field=models.DecimalField( - decimal_places=2, - help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10.", - max_digits=4, - null=True, - ), - ), - ] diff --git a/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py b/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py new file mode 100644 index 000000000..f7517070a --- /dev/null +++ b/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.2.16 on 2024-11-16 20:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0081_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="vulnerability", + name="exploitability", + field=models.DecimalField( + decimal_places=1, + help_text='"Exploitability indicates the likelihood that a vulnerability in a software package could be used \n by malicious actors to compromise systems, applications, or networks. \n This metric is determined automatically based on the discovery of known exploits.', + max_digits=2, + null=True, + ), + ), + migrations.AddField( + model_name="vulnerability", + name="weighted_severity", + field=models.DecimalField( + decimal_places=1, + help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10", + max_digits=3, + null=True, + ), + ), + migrations.AlterField( + model_name="package", + name="risk_score", + field=models.DecimalField( + decimal_places=1, + help_text="Risk score between 0.00 and 10.00, where higher values indicate greater vulnerability risk for the package.", + max_digits=3, + null=True, + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index c62949992..8cca1c021 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -243,6 +243,34 @@ class Vulnerability(models.Model): related_name="vulnerabilities", ) + exploitability = models.DecimalField( + null=True, + max_digits=2, + decimal_places=1, + help_text=""""Exploitability indicates the likelihood that a vulnerability in a software package could be used + by malicious actors to compromise systems, applications, or networks. + This metric is determined automatically based on the discovery of known exploits.""", + ) + + weighted_severity = models.DecimalField( + null=True, + max_digits=3, + decimal_places=1, + help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10", + ) + + @property + def risk_score(self): + """ + Risk expressed as a number ranging from 0 to 10. + Risk is calculated from weighted severity and exploitability values. + It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 + Risk = min(weighted severity * exploitability, 10) + """ + if self.exploitability and self.weighted_severity: + risk_score = min(float(self.exploitability * self.weighted_severity), 10.0) + return round(risk_score, 1) + objects = VulnerabilityQuerySet.as_manager() class Meta: @@ -672,8 +700,8 @@ class Package(PackageURLMixin): risk_score = models.DecimalField( null=True, - max_digits=4, - decimal_places=2, + max_digits=3, + decimal_places=1, help_text="Risk score between 0.00 and 10.00, where higher values " "indicate greater vulnerability risk for the package.", ) diff --git a/vulnerabilities/pipelines/compute_package_risk.py b/vulnerabilities/pipelines/compute_package_risk.py index 839d5241b..e8973b480 100644 --- a/vulnerabilities/pipelines/compute_package_risk.py +++ b/vulnerabilities/pipelines/compute_package_risk.py @@ -34,10 +34,11 @@ def steps(cls): ) def compute_and_store_vulnerability_risk_score(self): - affected_vulnerabilities = ( - Vulnerability.objects.filter(affectedbypackagerelatedvulnerability__isnull=False) - .prefetch_related("references") - .only("references", "exploits") + affected_vulnerabilities = Vulnerability.objects.filter( + affectedbypackagerelatedvulnerability__isnull=False + ).prefetch_related( + "references", + "exploits", ) self.log( @@ -51,8 +52,8 @@ def compute_and_store_vulnerability_risk_score(self): batch_size = 5000 for vulnerability in progress.iter(affected_vulnerabilities.paginated()): - references = vulnerability.references - severities = vulnerability.severities.select_related("reference") + severities = vulnerability.severities.all() + references = vulnerability.references.all() ( vulnerability.weighted_severity, @@ -76,17 +77,8 @@ def compute_and_store_vulnerability_risk_score(self): def compute_and_store_package_risk_score(self): affected_packages = ( - Package.objects.filter(affected_by_vulnerabilities__isnull=False).prefetch_related( - "affectedbypackagerelatedvulnerability_set__vulnerability", - "affectedbypackagerelatedvulnerability_set__vulnerability__references", - "affectedbypackagerelatedvulnerability_set__vulnerability__severities", - "affectedbypackagerelatedvulnerability_set__vulnerability__exploits", - ) - ).distinct() - - affected_packages = Package.objects.filter( - affected_by_vulnerabilities__isnull=False - ).distinct() + Package.objects.filter(affected_by_vulnerabilities__isnull=False).only("id").distinct() + ) self.log(f"Calculating risk for {affected_packages.count():,d} affected package records") diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index 5f5c480e5..6c28faa36 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -6,24 +6,22 @@ # See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # - - +from typing import List from urllib.parse import urlparse from django.db.models import Prefetch from vulnerabilities.models import AffectedByPackageRelatedVulnerability -from vulnerabilities.models import Exploit -from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.severity_systems import EPSS from vulnerabilities.weight_config import WEIGHT_CONFIG DEFAULT_WEIGHT = 5 -def get_weighted_severity(severities): +def get_weighted_severity(severities: List[VulnerabilitySeverity]): """ Weighted Severity is the maximum value obtained when each Severity is multiplied by its associated Weight/10. @@ -57,7 +55,9 @@ def get_weighted_severity(severities): vul_score_value = score_map.get(vul_score, 0) * max_weight score_list.append(vul_score_value) - return max(score_list) if score_list else 0 + + max_score = max(score_list) if score_list else 0 + return round(max_score, 1) def get_exploitability_level(exploits, references, severities): @@ -99,12 +99,8 @@ def compute_vulnerability_risk_factors(references, severities, exploits): Risk = min(weighted severity * exploitability, 10) """ - severities = severities.all() - exploits = exploits.all() - reference = references.all() - weighted_severity = get_weighted_severity(severities) - exploitability = get_exploitability_level(exploits, reference, severities) + exploitability = get_exploitability_level(exploits, references, severities) return weighted_severity, exploitability @@ -130,4 +126,4 @@ def compute_package_risk(package): if not result: return - return f"{max(result):.2f}" + return round(max(result), 1) diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py index 454fc3dd1..cbe1003b2 100644 --- a/vulnerabilities/tests/test_risk.py +++ b/vulnerabilities/tests/test_risk.py @@ -131,7 +131,7 @@ def test_exploitability_level( @pytest.mark.django_db def test_get_weighted_severity(vulnerability): severities = vulnerability.severities.all() - assert get_weighted_severity(severities) == 6.210000000000001 + assert get_weighted_severity(severities) == 6.2 severity2 = VulnerabilitySeverity.objects.create( url="https://security-tracker.debian.org/tracker/CVE-2019-13057", @@ -146,21 +146,17 @@ def test_get_weighted_severity(vulnerability): @pytest.mark.django_db def test_compute_vulnerability_risk_factors(vulnerability): - assert compute_vulnerability_risk_factors( - vulnerability.references, vulnerability.severities, vulnerability.exploits - ) == (6.210000000000001, 2) - assert compute_vulnerability_risk_factors( - vulnerability.references, vulnerability.severities, None - ) == ( - 6.210000000000001, - 0.5, - ) - assert compute_vulnerability_risk_factors( - vulnerability.references, None, vulnerability.exploits - ) == ( - 0, + severities = vulnerability.severities.all() + references = vulnerability.references.all() + + assert compute_vulnerability_risk_factors(references, severities, vulnerability.exploits) == ( + 6.2, 2, ) + + assert compute_vulnerability_risk_factors(references, severities, None) == (6.2, 0.5) + assert compute_vulnerability_risk_factors(references, None, vulnerability.exploits) == (0, 2) + assert compute_vulnerability_risk_factors(None, None, None) == (0, 0.5) @@ -169,15 +165,15 @@ def test_get_vulnerability_risk_score(vulnerability): vulnerability.weighted_severity = 6.0 vulnerability.exploitability = 2 - assert vulnerability.risk_score == "10" # max risk_score can be reached + assert vulnerability.risk_score == 10.0 # max risk_score can be reached vulnerability.weighted_severity = 6 vulnerability.exploitability = 0.5 - assert vulnerability.risk_score == "3" + assert vulnerability.risk_score == 3.0 vulnerability.weighted_severity = 5.6 vulnerability.exploitability = 0.5 - assert vulnerability.risk_score == "2.8" + assert vulnerability.risk_score == 2.8 vulnerability.weighted_severity = None vulnerability.exploitability = 0.5 From 893183f169a58ea73370c5900489029a527d2fa3 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Sun, 17 Nov 2024 15:53:35 +0200 Subject: [PATCH 5/8] Correct help text and remove inconsistent typing. Signed-off-by: ziad hany --- .../0082_vulnerability_exploitability_and_more.py | 6 +++--- vulnerabilities/models.py | 7 +++---- vulnerabilities/risk.py | 4 +--- vulnerabilities/templates/vulnerability_details.html | 6 +++--- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py b/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py index f7517070a..26a55e714 100644 --- a/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py +++ b/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.16 on 2024-11-16 20:41 +# Generated by Django 4.2.16 on 2024-11-17 13:52 from django.db import migrations, models @@ -15,7 +15,7 @@ class Migration(migrations.Migration): name="exploitability", field=models.DecimalField( decimal_places=1, - help_text='"Exploitability indicates the likelihood that a vulnerability in a software package could be used \n by malicious actors to compromise systems, applications, or networks. \n This metric is determined automatically based on the discovery of known exploits.', + help_text="Exploitability indicates the likelihood that a vulnerability in a software package could be used by malicious actors to compromise systems, applications, or networks. This metric is determined automatically based on the discovery of known exploits.", max_digits=2, null=True, ), @@ -25,7 +25,7 @@ class Migration(migrations.Migration): name="weighted_severity", field=models.DecimalField( decimal_places=1, - help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10", + help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10.", max_digits=3, null=True, ), diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 8cca1c021..e5fe231f6 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -247,16 +247,15 @@ class Vulnerability(models.Model): null=True, max_digits=2, decimal_places=1, - help_text=""""Exploitability indicates the likelihood that a vulnerability in a software package could be used - by malicious actors to compromise systems, applications, or networks. - This metric is determined automatically based on the discovery of known exploits.""", + help_text="Exploitability indicates the likelihood that a vulnerability in a software package could be used by malicious actors to compromise systems, " + "applications, or networks. This metric is determined automatically based on the discovery of known exploits.", ) weighted_severity = models.DecimalField( null=True, max_digits=3, decimal_places=1, - help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10", + help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10.", ) @property diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index 6c28faa36..8ccbd0e3d 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -6,7 +6,6 @@ # See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -from typing import List from urllib.parse import urlparse from django.db.models import Prefetch @@ -14,14 +13,13 @@ from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.severity_systems import EPSS from vulnerabilities.weight_config import WEIGHT_CONFIG DEFAULT_WEIGHT = 5 -def get_weighted_severity(severities: List[VulnerabilitySeverity]): +def get_weighted_severity(severities): """ Weighted Severity is the maximum value obtained when each Severity is multiplied by its associated Weight/10. diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 7381c058c..e9e58c79e 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -124,9 +124,9 @@ + data-tooltip="Exploitability indicates the likelihood that a vulnerability in a software package + could be used by malicious actors to compromise systems, + applications, or networks. This metric is determined automatically based on the discovery of known exploits."> Exploitability {{ vulnerability.exploitability }} From f29ef16a91c1b53d9bdb436dfc9d91525dea7949 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Mon, 18 Nov 2024 16:21:25 +0200 Subject: [PATCH 6/8] Add severities in the prefetch and optimize the prefetching process for compute_and_store_package_risk_score Signed-off-by: ziad hany --- .../pipelines/compute_package_risk.py | 13 +++++++++++-- vulnerabilities/risk.py | 18 +++--------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/vulnerabilities/pipelines/compute_package_risk.py b/vulnerabilities/pipelines/compute_package_risk.py index e8973b480..22b3305a6 100644 --- a/vulnerabilities/pipelines/compute_package_risk.py +++ b/vulnerabilities/pipelines/compute_package_risk.py @@ -8,6 +8,7 @@ # from aboutcode.pipeline import LoopProgress +from django.db.models import Prefetch from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability @@ -38,6 +39,7 @@ def compute_and_store_vulnerability_risk_score(self): affectedbypackagerelatedvulnerability__isnull=False ).prefetch_related( "references", + "severities", "exploits", ) @@ -77,8 +79,15 @@ def compute_and_store_vulnerability_risk_score(self): def compute_and_store_package_risk_score(self): affected_packages = ( - Package.objects.filter(affected_by_vulnerabilities__isnull=False).only("id").distinct() - ) + Package.objects.filter(affected_by_vulnerabilities__isnull=False) + .only("id") + .prefetch_related( + Prefetch( + "affectedbypackagerelatedvulnerability_set__vulnerability", + queryset=Vulnerability.objects.only("weighted_severity", "exploitability"), + ), + ) + ).distinct() self.log(f"Calculating risk for {affected_packages.count():,d} affected package records") diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index 8ccbd0e3d..e6306a704 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -8,10 +8,6 @@ # from urllib.parse import urlparse -from django.db.models import Prefetch - -from vulnerabilities.models import AffectedByPackageRelatedVulnerability -from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.severity_systems import EPSS from vulnerabilities.weight_config import WEIGHT_CONFIG @@ -107,18 +103,10 @@ def compute_package_risk(package): Calculate the risk for a package by iterating over all vulnerabilities that affects this package and determining the associated risk. """ - result = [] - affected_pkg_related_vul = AffectedByPackageRelatedVulnerability.objects.filter( - package=package - ).prefetch_related( - Prefetch( - "vulnerability", - queryset=Vulnerability.objects.only("weighted_severity", "exploitability"), - ) - ) - for pkg_related_vul in affected_pkg_related_vul: - if risk := pkg_related_vul.vulnerability.risk_score: + vulnerabilities = package.vulnerabilities.all() + for vulnerability in vulnerabilities: + if risk := vulnerability.risk_score: result.append(float(risk)) if not result: From 4920e1f71b6c6f8f05a24a3aeefbc3d3f00a3a58 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Mon, 18 Nov 2024 17:45:00 +0200 Subject: [PATCH 7/8] Refactor the risk score calculation for vulnerabilities and packages. Update the tests for exploits and the simple_risk_pipeline. Signed-off-by: ziad hany --- .../pipelines/compute_package_risk.py | 89 +++++++++---------- vulnerabilities/risk.py | 5 +- .../pipelines/test_compute_package_risk.py | 2 +- vulnerabilities/tests/test_risk.py | 7 +- 4 files changed, 51 insertions(+), 52 deletions(-) diff --git a/vulnerabilities/pipelines/compute_package_risk.py b/vulnerabilities/pipelines/compute_package_risk.py index 22b3305a6..7ac4de838 100644 --- a/vulnerabilities/pipelines/compute_package_risk.py +++ b/vulnerabilities/pipelines/compute_package_risk.py @@ -6,7 +6,6 @@ # See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # - from aboutcode.pipeline import LoopProgress from django.db.models import Prefetch @@ -35,12 +34,14 @@ def steps(cls): ) def compute_and_store_vulnerability_risk_score(self): - affected_vulnerabilities = Vulnerability.objects.filter( - affectedbypackagerelatedvulnerability__isnull=False - ).prefetch_related( - "references", - "severities", - "exploits", + affected_vulnerabilities = ( + Vulnerability.objects.filter(affecting_packages__isnull=False) + .prefetch_related( + "references", + "severities", + "exploits", + ) + .distinct() ) self.log( @@ -53,35 +54,43 @@ def compute_and_store_vulnerability_risk_score(self): updated_vulnerability_count = 0 batch_size = 5000 - for vulnerability in progress.iter(affected_vulnerabilities.paginated()): + for vulnerability in progress.iter(affected_vulnerabilities.paginated(per_page=batch_size)): severities = vulnerability.severities.all() references = vulnerability.references.all() + exploits = vulnerability.exploits.all() - ( - vulnerability.weighted_severity, - vulnerability.exploitability, - ) = compute_vulnerability_risk_factors(references, severities, vulnerability.exploits) + weighted_severity, exploitability = compute_vulnerability_risk_factors( + references=references, + severities=severities, + exploits=exploits, + ) + vulnerability.weighted_severity = weighted_severity + vulnerability.exploitability = exploitability updatables.append(vulnerability) if len(updatables) >= batch_size: - updated_vulnerability_count += bulk_update_vulnerability_risk_score( - vulnerabilities=updatables, + updated_vulnerability_count += bulk_update( + model=Vulnerability, + items=updatables, + fields=["weighted_severity", "exploitability"], logger=self.log, ) - updated_vulnerability_count += bulk_update_vulnerability_risk_score( - vulnerabilities=updatables, + + updated_vulnerability_count += bulk_update( + model=Vulnerability, + items=updatables, + fields=["weighted_severity", "exploitability"], logger=self.log, ) + self.log( f"Successfully added risk score for {updated_vulnerability_count:,d} vulnerability" ) def compute_and_store_package_risk_score(self): affected_packages = ( - Package.objects.filter(affected_by_vulnerabilities__isnull=False) - .only("id") - .prefetch_related( + Package.objects.filter(affected_by_vulnerabilities__isnull=False).prefetch_related( Prefetch( "affectedbypackagerelatedvulnerability_set__vulnerability", queryset=Vulnerability.objects.only("weighted_severity", "exploitability"), @@ -111,38 +120,28 @@ def compute_and_store_package_risk_score(self): updatables.append(package) if len(updatables) >= batch_size: - updated_package_count += bulk_update_package_risk_score( - packages=updatables, + updated_package_count += bulk_update( + model=Package, + items=updatables, + fields=["risk_score"], logger=self.log, ) - updated_package_count += bulk_update_package_risk_score( - packages=updatables, + updated_package_count += bulk_update( + model=Package, + items=updatables, + fields=["risk_score"], logger=self.log, ) self.log(f"Successfully added risk score for {updated_package_count:,d} package") -def bulk_update_package_risk_score(packages, logger): - package_count = 0 - if packages: - try: - Package.objects.bulk_update(objs=packages, fields=["risk_score"]) - package_count += len(packages) - except Exception as e: - logger(f"Error updating packages: {e}") - packages.clear() - return package_count - - -def bulk_update_vulnerability_risk_score(vulnerabilities, logger): - vulnerabilities_count = 0 - if vulnerabilities: +def bulk_update(model, items, fields, logger): + item_count = 0 + if items: try: - Vulnerability.objects.bulk_update( - objs=vulnerabilities, fields=["weighted_severity", "exploitability"] - ) - vulnerabilities_count += len(vulnerabilities) + model.objects.bulk_update(objs=items, fields=fields) + item_count += len(items) except Exception as e: - logger(f"Error updating vulnerability: {e}") - vulnerabilities.clear() - return vulnerabilities_count + logger(f"Error updating {model.__name__}: {e}") + items.clear() + return item_count diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index e6306a704..b38d50a4e 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -104,9 +104,8 @@ def compute_package_risk(package): and determining the associated risk. """ result = [] - vulnerabilities = package.vulnerabilities.all() - for vulnerability in vulnerabilities: - if risk := vulnerability.risk_score: + for vulnerability in package.affectedbypackagerelatedvulnerability_set.all(): + if risk := vulnerability.vulnerability.risk_score: result.append(float(risk)) if not result: diff --git a/vulnerabilities/tests/pipelines/test_compute_package_risk.py b/vulnerabilities/tests/pipelines/test_compute_package_risk.py index b8608cbc0..a366d32e8 100644 --- a/vulnerabilities/tests/pipelines/test_compute_package_risk.py +++ b/vulnerabilities/tests/pipelines/test_compute_package_risk.py @@ -31,4 +31,4 @@ def test_simple_risk_pipeline(vulnerability): improver.execute() pkg = Package.objects.get(type="pypi", name="foo", version="2.3.0") - assert pkg.risk_score == Decimal("10") + assert pkg.risk_score == Decimal("3.1") # max( 6.9 * 9/10 , 6.5 * 9/10 ) * .5 = 3.105 diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py index cbe1003b2..420c8c402 100644 --- a/vulnerabilities/tests/test_risk.py +++ b/vulnerabilities/tests/test_risk.py @@ -145,17 +145,18 @@ def test_get_weighted_severity(vulnerability): @pytest.mark.django_db -def test_compute_vulnerability_risk_factors(vulnerability): +def test_compute_vulnerability_risk_factors(vulnerability, exploit): severities = vulnerability.severities.all() references = vulnerability.references.all() - assert compute_vulnerability_risk_factors(references, severities, vulnerability.exploits) == ( + assert compute_vulnerability_risk_factors(references, severities, exploit) == ( 6.2, 2, ) assert compute_vulnerability_risk_factors(references, severities, None) == (6.2, 0.5) - assert compute_vulnerability_risk_factors(references, None, vulnerability.exploits) == (0, 2) + + assert compute_vulnerability_risk_factors(references, None, exploit) == (0, 2) assert compute_vulnerability_risk_factors(None, None, None) == (0, 0.5) From 1a9df9b6501f5f5d270eacf3c61827eae6d3a579 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Mon, 18 Nov 2024 21:28:20 +0200 Subject: [PATCH 8/8] Rename vulnerability to relation for clarity in compute_package_risk Signed-off-by: ziad hany --- vulnerabilities/risk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index b38d50a4e..a4508a03f 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -104,8 +104,8 @@ def compute_package_risk(package): and determining the associated risk. """ result = [] - for vulnerability in package.affectedbypackagerelatedvulnerability_set.all(): - if risk := vulnerability.vulnerability.risk_score: + for relation in package.affectedbypackagerelatedvulnerability_set.all(): + if risk := relation.vulnerability.risk_score: result.append(float(risk)) if not result: