diff --git a/vulnerabilities/importers/safety_db.py b/vulnerabilities/importers/safety_db.py deleted file mode 100755 index 81a19b006..000000000 --- a/vulnerabilities/importers/safety_db.py +++ /dev/null @@ -1,144 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import asyncio -import logging -import re -from typing import Any -from typing import Iterable -from typing import Mapping -from typing import Set -from typing import Tuple - -import requests -from packageurl import PackageURL -from univers.version_range import PypiVersionRange -from univers.versions import InvalidVersion -from univers.versions import PypiVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.package_managers import PypiVersionAPI -from vulnerabilities.utils import nearest_patched_package - -logger = logging.getLogger(__name__) - - -class SafetyDbImporter(Importer): - def __enter__(self): - self._api_response = self._fetch() - self._versions = PypiVersionAPI() - self.set_api(self.collect_packages()) - - @property - def versions(self): # quick hack to make it patchable - return self._versions - - def set_api(self, packages): - asyncio.run(self._versions.load_api(packages)) - - def _fetch(self) -> Mapping[str, Any]: - if self.create_etag(self.config.url): - return requests.get(self.config.url).json() - return [] - - def collect_packages(self): - return {pkg for pkg in self._api_response} - - def updated_advisories(self) -> Set[AdvisoryData]: - for package_name in self._api_response: - if package_name == "$meta" or package_name == "cumin": - # This is the first entry in the data feed. It contains metadata of the feed. - # Skip it. The 'cumin' entry is wrong - continue - - all_package_versions = self.versions.get(package_name).valid_versions - if not len(all_package_versions): - # PyPi does not have data about this package, we skip these - continue - - for advisory in self._api_response[package_name]: - if advisory["cve"]: - # Check on advisory["cve"] instead of using `get` because it can have null value - cve_ids = re.findall(r"CVE-\d+-\d+", advisory["cve"]) - else: - continue - - impacted_purls, resolved_purls = categorize_versions( - package_name, all_package_versions, advisory["specs"] - ) - - reference = [Reference(reference_id=advisory["id"])] - advisories = [] - for cve_id in cve_ids: - advisories.append( - AdvisoryData( - vulnerability_id=cve_id, - summary=advisory["advisory"], - references=reference, - affected_packages=nearest_patched_package( - impacted_purls, resolved_purls - ), - ) - ) - - yield advisories - - # FIXME: This is duplicate code. Use the the helper instead. - def create_etag(self, url): - etag = requests.head(url).headers.get("ETag") - if not etag: - # Kind of inaccurate to return True since etag is - # not created - return True - elif url in self.config.etags: - if self.config.etags[url] == etag: - return False - self.config.etags[url] = etag - return True - - -# FIXME: This function is horribly named incorretly. -def categorize_versions( - package_name: str, - all_versions: Set[str], - version_specs: Iterable[str], -) -> Tuple[Set[PackageURL], Set[PackageURL]]: - """ - :return: impacted, resolved purls - """ - impacted_versions, impacted_purls = set(), [] - vurl_specs = [] - for version_spec in version_specs: - vurl_specs.append(PypiVersionRange.from_native(version_spec)) - - invalid_versions = set() - for version in all_versions: - try: - version_object = PypiVersion(version) - except InvalidVersion: - invalid_versions.add(version) - continue - - if any([version_object in vurl_spec for vurl_spec in vurl_specs]): - impacted_versions.add(version) - impacted_purls.append( - PackageURL( - name=package_name, - type="pypi", - version=version, - ) - ) - - resolved_purls = [] - all_versions -= invalid_versions - for version in all_versions - impacted_versions: - resolved_purls.append(PackageURL(name=package_name, type="pypi", version=version)) - return impacted_purls, resolved_purls diff --git a/vulnerabilities/tests/conftest.py b/vulnerabilities/tests/conftest.py index c692c2a80..b2bb96038 100644 --- a/vulnerabilities/tests/conftest.py +++ b/vulnerabilities/tests/conftest.py @@ -31,7 +31,6 @@ def no_rmtree(monkeypatch): "test_package_managers.py", "test_ruby.py", "test_rust.py", - "test_safety_db.py", "test_suse_backports.py", "test_suse.py", "test_ubuntu_usn.py", diff --git a/vulnerabilities/tests/test_data/safety_db/insecure_full.json b/vulnerabilities/tests/test_data/safety_db/insecure_full.json deleted file mode 100644 index 770270ff8..000000000 --- a/vulnerabilities/tests/test_data/safety_db/insecure_full.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "django": [ - { - "advisory": "The utils.http.is_safe_url function in Django before 1.4.20, 1.5.x, 1.6.x before 1.6.11, 1.7.x before 1.7.7, and 1.8.x before 1.8c1 does not properly validate URLs, which allows remote attackers to conduct cross-site scripting (XSS) attacks via a control character in a URL, as demonstrated by a \\x08javascript: URL.", - "cve": "CVE-2015-2317", - "id": "pyup.io-25713", - "specs": [ - ">=1.8,<1.8.2", - "<1.4.20", - ">=1.5,<1.6", - ">=1.6,<1.6.11", - ">=1.7,<1.7.7" - ], - "v": ">=1.8,<1.8.2,<1.4.20,>=1.5,<1.6,>=1.6,<1.6.11,>=1.7,<1.7.7" - }, - { - "advisory": "Cross-site scripting (XSS) vulnerability in the dismissChangeRelatedObjectPopup function in contrib/admin/static/admin/js/admin/RelatedObjectLookups.js in Django before 1.8.14, 1.9.x before 1.9.8, and 1.10.x before 1.10rc1 allows remote attackers to inject arbitrary web script or HTML via vectors involving unsafe usage of Element.innerHTML.", - "cve": "CVE-2016-6186", - "id": "pyup.io-25721", - "specs": [ - ">=1.9,<1.9.8", - "==1.8.14", - ">=1.10,<1.10.2" - ], - "v": ">=1.9,<1.9.8,==1.8.14,>=1.10,<1.10.2" - } - ] -} diff --git a/vulnerabilities/tests/test_safety_db.py b/vulnerabilities/tests/test_safety_db.py deleted file mode 100644 index 7749b92ef..000000000 --- a/vulnerabilities/tests/test_safety_db.py +++ /dev/null @@ -1,176 +0,0 @@ -# -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -from unittest import TestCase - -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Reference -from vulnerabilities.importers.safety_db import SafetyDbImporter -from vulnerabilities.importers.safety_db import categorize_versions -from vulnerabilities.package_managers import LegacyPypiVersionAPI -from vulnerabilities.package_managers import Version -from vulnerabilities.utils import AffectedPackage - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "safety_db") - -MOCK_VERSION_API = LegacyPypiVersionAPI( - cache={ - "ampache": {Version("2.0"), Version("5.2.1")}, - "django": { - Version("1.8"), - Version("1.4.19"), - Version("1.4.22"), - Version("1.5.1"), - Version("1.6.9"), - Version("1.8.14"), - }, - "zulip": {Version("2.0"), Version("2.1.1"), Version("2.1.2"), Version("2.1.3")}, - } -) - - -class SafetyDbtTest(TestCase): - def test_import(self): - data_src = SafetyDbImporter(1, config={"url": "https://gmail.com/", "etags": ""}) - with open(os.path.join(TEST_DATA, "insecure_full.json")) as f: - raw_data = json.load(f) - data_src._api_response = raw_data - data_src._versions = MOCK_VERSION_API - - expected_data = [ - AdvisoryData( - summary="The utils.http.is_safe_url function in Django before 1.4.20, 1.5.x, 1.6.x before 1.6.11, 1.7.x before 1.7.7, and 1.8.x before 1.8c1 does not properly validate URLs, which allows remote attackers to conduct cross-site scripting (XSS) attacks via a control character in a URL, as demonstrated by a \\x08javascript: URL.", - vulnerability_id="CVE-2015-2317", - affected_packages=[ - AffectedPackage( - vulnerable_package=PackageURL( - type="pypi", - namespace=None, - name="django", - version="1.4.19", - qualifiers={}, - subpath=None, - ), - patched_package=PackageURL( - type="pypi", - namespace=None, - name="django", - version="1.4.22", - qualifiers={}, - subpath=None, - ), - ), - AffectedPackage( - vulnerable_package=PackageURL( - type="pypi", - namespace=None, - name="django", - version="1.5.1", - qualifiers={}, - subpath=None, - ), - patched_package=PackageURL( - type="pypi", - namespace=None, - name="django", - version="1.8.14", - qualifiers={}, - subpath=None, - ), - ), - AffectedPackage( - vulnerable_package=PackageURL( - type="pypi", - namespace=None, - name="django", - version="1.6.9", - qualifiers={}, - subpath=None, - ), - patched_package=PackageURL( - type="pypi", - namespace=None, - name="django", - version="1.8.14", - qualifiers={}, - subpath=None, - ), - ), - AffectedPackage( - vulnerable_package=PackageURL( - type="pypi", - namespace=None, - name="django", - version="1.8", - qualifiers={}, - subpath=None, - ), - patched_package=PackageURL( - type="pypi", - namespace=None, - name="django", - version="1.8.14", - qualifiers={}, - subpath=None, - ), - ), - ], - references=[Reference(reference_id="pyup.io-25713", url="", severities=[])], - ), - AdvisoryData( - summary="Cross-site scripting (XSS) vulnerability in the dismissChangeRelatedObjectPopup function in contrib/admin/static/admin/js/admin/RelatedObjectLookups.js in Django before 1.8.14, 1.9.x before 1.9.8, and 1.10.x before 1.10rc1 allows remote attackers to inject arbitrary web script or HTML via vectors involving unsafe usage of Element.innerHTML.", - vulnerability_id="CVE-2016-6186", - affected_packages=[ - AffectedPackage( - vulnerable_package=PackageURL( - type="pypi", - namespace=None, - name="django", - version="1.8.14", - qualifiers={}, - subpath=None, - ), - patched_package=None, - ) - ], - references=[Reference(reference_id="pyup.io-25721", url="", severities=[])], - ), - ] - - found_data = [] - # FIXME: This is messed up - for adv_batch in data_src.updated_advisories(): - found_data.extend(adv_batch) - # found_data = [list(adv) for adv in data_src.updated_advisories()] - - # print(expected_data) - # print("\n", found_data) - assert expected_data == found_data - - -def test_categorize_versions(): - all_versions = {"1.8", "1.4.19", "1.4.22", "1.5.1", "1.6.9", "1.8.14"} - version_specs = [">=1.8,<1.8.3", "<1.4.20", ">=1.5,<1.6", ">=1.6,<1.6.11", ">=1.7,<1.7.7"] - - impacted_purls, resolved_purls = categorize_versions("django", all_versions, version_specs) - - assert len(impacted_purls) == 4 - assert len(resolved_purls) == 2 - - impacted_versions = {p.version for p in impacted_purls} - resolved_versions = {p.version for p in resolved_purls} - - assert impacted_versions == {"1.8", "1.4.19", "1.5.1", "1.6.9"} - assert resolved_versions == {"1.4.22", "1.8.14"}