From 8cee434625326411c95b42c207592c70ebe84557 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Mon, 25 Dec 2023 15:36:00 +0530 Subject: [PATCH 01/62] Add progress bar to NVDImporter Signed-off-by: Harsh Mishra --- requirements.txt | 1 + vulnerabilities/importers/nvd.py | 15 ++++++++++++--- vulnerabilities/management/commands/import.py | 11 ++++++++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 042f541da..7a26d552b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -120,3 +120,4 @@ drf-spectacular==0.24.2 coreapi==2.3.3 coreschema==0.0.4 itypes==1.2.0 +progress==1.6 \ No newline at end of file diff --git a/vulnerabilities/importers/nvd.py b/vulnerabilities/importers/nvd.py index b22d3db68..3c72d0206 100644 --- a/vulnerabilities/importers/nvd.py +++ b/vulnerabilities/importers/nvd.py @@ -10,6 +10,7 @@ import gzip import json from datetime import date +from progress.bar import ChargingBar import attr import requests @@ -78,10 +79,18 @@ def fetch_cve_data_1_1(starting_year=2002): year since ``starting_year`` defaulting to 2002. """ current_year = date.today().year + progress_bar_for_records_fetched = ChargingBar("\tRecords fetched", max=(current_year-starting_year)+1) + progress_bar_for_records_fetched.start() # NVD json feeds start from 2002. - for year in range(starting_year, current_year + 1): - download_url = f"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{year}.json.gz" - yield year, fetch(url=download_url) + try: + for year in range(starting_year, current_year + 1): + try: + download_url = f"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{year}.json.gz" + yield year, fetch(url=download_url) + finally: + progress_bar_for_records_fetched.next() + finally: + progress_bar_for_records_fetched.finish() def to_advisories(cve_data): diff --git a/vulnerabilities/management/commands/import.py b/vulnerabilities/management/commands/import.py index 5ae885299..9bbb2394f 100644 --- a/vulnerabilities/management/commands/import.py +++ b/vulnerabilities/management/commands/import.py @@ -8,9 +8,10 @@ # import traceback +import progress from django.core.management.base import BaseCommand from django.core.management.base import CommandError - +from progress.bar import IncrementalBar from vulnerabilities.import_runner import ImportRunner from vulnerabilities.importers import IMPORTERS_REGISTRY @@ -54,9 +55,10 @@ def import_data(self, importers): names for the importers. """ failed_importers = [] - + progress_bar_for_import = IncrementalBar("Fetching Data from Databases", max=len(importers)) + progress_bar_for_import.start() for importer in importers: - self.stdout.write(f"Importing data using {importer.qualified_name}") + self.stdout.write(f"\nImporting data using {importer.qualified_name}") try: ImportRunner(importer).run() self.stdout.write( @@ -72,6 +74,9 @@ def import_data(self, importers): f"Failed to run importer {importer.qualified_name}. Continuing..." ) ) + finally: + progress_bar_for_import.next() + progress_bar_for_import.finish() if failed_importers: raise CommandError(f"{len(failed_importers)} failed!: {','.join(failed_importers)}") From 21ec9f916ed593fe40c9312da0ac880fdff7b713 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 26 Dec 2023 15:12:50 +0530 Subject: [PATCH 02/62] Add progress bar to GithubImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/github.py | 48 +++++++++++++++++++---------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py index 8ef1b3a9c..c3019ebc8 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/importers/github.py @@ -14,6 +14,7 @@ from cwe2.database import Database from dateutil import parser as dateparser from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import build_range_from_github_advisory_constraint @@ -88,27 +89,42 @@ } """ +progress_bar_for_package_fetch = ChargingBar( + "\tFetching Packages", max=len(PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items()) +) + class GitHubAPIImporter(Importer): spdx_license_expression = "CC-BY-4.0" def advisory_data(self) -> Iterable[AdvisoryData]: - for ecosystem, package_type in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items(): - end_cursor_exp = "" - while True: - graphql_query = {"query": GRAPHQL_QUERY_TEMPLATE % (ecosystem, end_cursor_exp)} - response = utils.fetch_github_graphql_query(graphql_query) - - page_info = get_item(response, "data", "securityVulnerabilities", "pageInfo") - end_cursor = get_item(page_info, "endCursor") - if end_cursor: - end_cursor = f'"{end_cursor}"' - end_cursor_exp = f"after: {end_cursor}" - - yield from process_response(response, package_type=package_type) - - if not get_item(page_info, "hasNextPage"): - break + progress_bar_for_package_fetch.start() + try: + for ecosystem, package_type in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items(): + yield from send_graphql_query(ecosystem, package_type) + finally: + progress_bar_for_package_fetch.finish() + + +def send_graphql_query(ecosystem, package_type) -> Iterable[AdvisoryData]: + try: + end_cursor_exp = "" + while True: + graphql_query = {"query": GRAPHQL_QUERY_TEMPLATE % (ecosystem, end_cursor_exp)} + response = utils.fetch_github_graphql_query(graphql_query) + + page_info = get_item(response, "data", "securityVulnerabilities", "pageInfo") + end_cursor = get_item(page_info, "endCursor") + if end_cursor: + end_cursor = f'"{end_cursor}"' + end_cursor_exp = f"after: {end_cursor}" + + yield from process_response(response, package_type=package_type) + + if not get_item(page_info, "hasNextPage"): + break + finally: + progress_bar_for_package_fetch.next() def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]: From 57e0e286126eaed567cbeb11fd8bd409050c8f1e Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 27 Dec 2023 21:57:10 +0530 Subject: [PATCH 03/62] Add missing type annotations to GithubImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py index c3019ebc8..b267fe5e6 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/importers/github.py @@ -106,7 +106,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]: progress_bar_for_package_fetch.finish() -def send_graphql_query(ecosystem, package_type) -> Iterable[AdvisoryData]: +def send_graphql_query(ecosystem: str, package_type: str) -> Iterable[AdvisoryData]: try: end_cursor_exp = "" while True: From c1b06ed2eee1304c5740ccc7e67b00e343226503 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Thu, 28 Dec 2023 01:12:47 +0530 Subject: [PATCH 04/62] Add progress bar for NPMImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/npm.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index 4dcc30705..b8df1478e 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -16,6 +16,7 @@ import pytz from dateutil.parser import parse from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_range import NpmVersionRange from vulnerabilities.importer import AdvisoryData @@ -36,15 +37,23 @@ class NpmImporter(Importer): importer_name = "Npm Importer" def advisory_data(self) -> Iterable[AdvisoryData]: + progress_bar_for_fetched_files = ChargingBar("\tFetching Files") try: self.clone(self.repo_url) path = Path(self.vcs_response.dest_dir) - vuln = path / "vuln" npm_vulns = vuln / "npm" - for file in npm_vulns.glob("*.json"): - yield from self.to_advisory_data(file) + paths_for_files_fetched = list(npm_vulns.glob("*.json")) + progress_bar_for_fetched_files.max = len(paths_for_files_fetched) + progress_bar_for_fetched_files.start() + + for file in paths_for_files_fetched: + try: + yield from self.to_advisory_data(file) + finally: + progress_bar_for_fetched_files.next() finally: + progress_bar_for_fetched_files.finish() if self.vcs_response: self.vcs_response.delete() From 1de5e2598e066486f359f72bb47e7fc53536a870 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Thu, 28 Dec 2023 04:12:56 +0530 Subject: [PATCH 05/62] Add Progress bar for GitlabImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/gitlab.py | 34 ++++++++++++++++++----------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/importers/gitlab.py index cd42b24ed..2228fd97a 100644 --- a/vulnerabilities/importers/gitlab.py +++ b/vulnerabilities/importers/gitlab.py @@ -18,6 +18,7 @@ import saneyaml from dateutil import parser as dateparser from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import VersionRange from univers.version_range import from_gitlab_native @@ -55,23 +56,30 @@ class GitLabAPIImporter(Importer): repo_url = "git+https://gitlab.com/gitlab-org/advisories-community/" def advisory_data(self, _keep_clone=False) -> Iterable[AdvisoryData]: + progress_bar_for_package_fetch = ChargingBar("\tFetching Packages") try: self.clone(repo_url=self.repo_url) base_path = Path(self.vcs_response.dest_dir) - - for file_path in base_path.glob("**/*.yml"): - gitlab_type, package_slug, vuln_id = parse_advisory_path( - base_path=base_path, - file_path=file_path, - ) - - if gitlab_type in PURL_TYPE_BY_GITLAB_SCHEME: - yield parse_gitlab_advisory(file=file_path, base_path=base_path) - - else: - logger.error(f"Unknow package type {gitlab_type!r} in {file_path!r}") - continue + file_paths_for_fetched_files = list(base_path.glob("**/*.yml")) + progress_bar_for_package_fetch.max = len(file_paths_for_fetched_files) + progress_bar_for_package_fetch.start() + for file_path in file_paths_for_fetched_files: + try: + gitlab_type, package_slug, vuln_id = parse_advisory_path( + base_path=base_path, + file_path=file_path, + ) + + if gitlab_type in PURL_TYPE_BY_GITLAB_SCHEME: + yield parse_gitlab_advisory(file=file_path, base_path=base_path) + + else: + logger.error(f"Unknow package type {gitlab_type!r} in {file_path!r}") + continue + finally: + progress_bar_for_package_fetch.next() finally: + progress_bar_for_package_fetch.finish() if self.vcs_response and not _keep_clone: self.vcs_response.delete() From 38eeb69794ca4095d4f44b0fc53337f02e0f4a65 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Thu, 28 Dec 2023 04:28:37 +0530 Subject: [PATCH 06/62] Add Progress bar to PyPaImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/pypa.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/importers/pypa.py b/vulnerabilities/importers/pypa.py index 0f545be55..19173dddb 100644 --- a/vulnerabilities/importers/pypa.py +++ b/vulnerabilities/importers/pypa.py @@ -12,6 +12,7 @@ from typing import Iterable import saneyaml +from progress.bar import ChargingBar from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer @@ -28,16 +29,24 @@ class PyPaImporter(Importer): importer_name = "Pypa Importer" def advisory_data(self) -> Iterable[AdvisoryData]: + progress_bar_for_package_fetch = ChargingBar("\tFetching Packages") try: vcs_response = self.clone(repo_url=self.repo_url) path = Path(vcs_response.dest_dir) + progress_bar_for_package_fetch.max = len(dict(fork_and_get_files(base_path=path))) + progress_bar_for_package_fetch.start() + for advisory_url, raw_data in fork_and_get_files(base_path=path): - yield parse_advisory_data( - raw_data=raw_data, - supported_ecosystem="pypi", - advisory_url=advisory_url, - ) + try: + yield parse_advisory_data( + raw_data=raw_data, + supported_ecosystem="pypi", + advisory_url=advisory_url, + ) + finally: + progress_bar_for_package_fetch.next() finally: + progress_bar_for_package_fetch.finish() if self.vcs_response: self.vcs_response.delete() From e145df60e91ffe8f0f04ff42a3148b5e0950d9a3 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Thu, 28 Dec 2023 04:42:51 +0530 Subject: [PATCH 07/62] Add Progress bar to Nginx Importer Signed-off-by: Harsh Mishra --- vulnerabilities/importers/nginx.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/importers/nginx.py b/vulnerabilities/importers/nginx.py index 4fe0ca6ae..0645e47c3 100644 --- a/vulnerabilities/importers/nginx.py +++ b/vulnerabilities/importers/nginx.py @@ -16,6 +16,7 @@ from bs4 import BeautifulSoup from django.db.models.query import QuerySet from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_range import NginxVersionRange from univers.versions import NginxVersion @@ -52,10 +53,17 @@ def advisory_data_from_text(text): """ soup = BeautifulSoup(text, features="lxml") vuln_list = soup.select("li p") - for vuln_info in vuln_list: - ngnix_adv = parse_advisory_data_from_paragraph(vuln_info) - yield to_advisory_data(ngnix_adv) - + progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(vuln_list)) + try: + progress_bar_for_package_fetch.start() + for vuln_info in vuln_list: + try: + ngnix_adv = parse_advisory_data_from_paragraph(vuln_info) + yield to_advisory_data(ngnix_adv) + finally: + progress_bar_for_package_fetch.next() + finally: + progress_bar_for_package_fetch.finish() class NginxAdvisory(NamedTuple): aliases: list From 7b16b3958cdeb09dba29739ca221a53b382d50c5 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Thu, 28 Dec 2023 05:05:05 +0530 Subject: [PATCH 08/62] Add Progress bar to PyPIImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/pysec.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/pysec.py b/vulnerabilities/importers/pysec.py index b42d7cb7e..52e03283c 100644 --- a/vulnerabilities/importers/pysec.py +++ b/vulnerabilities/importers/pysec.py @@ -13,13 +13,14 @@ from zipfile import ZipFile import requests +from progress.bar import ChargingBar from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importers.osv import parse_advisory_data logger = logging.getLogger(__name__) - +progress_bar_for_package_fetch = ChargingBar("\tFetching Packages") class PyPIImporter(Importer): license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" @@ -33,7 +34,15 @@ def advisory_data(self) -> Iterable[AdvisoryData]: url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip" response = requests.get(url).content with ZipFile(BytesIO(response)) as zip_file: - for file_name in zip_file.namelist(): + progress_bar_for_package_fetch.max = len(zip_file.namelist()) + yield from process_zipfile_response(zip_file, url) + + +def process_zipfile_response(zip_file: ZipFile, url: str) -> Iterable[AdvisoryData]: + try: + progress_bar_for_package_fetch.start() + for file_name in zip_file.namelist(): + try: if not file_name.startswith("PYSEC-"): logger.error(f"Unsupported PyPI advisory data file: {file_name}") continue @@ -42,3 +51,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]: yield parse_advisory_data( raw_data=vul_info, supported_ecosystem="pypi", advisory_url=url ) + finally: + progress_bar_for_package_fetch.next() + finally: + progress_bar_for_package_fetch.finish() \ No newline at end of file From 1c8f6fc4b02d1210b329e0d38282215084a4ff39 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sat, 30 Dec 2023 00:07:39 +0530 Subject: [PATCH 09/62] Add Progress bar to AlpineImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/alpine_linux.py | 33 ++++++++++++++--------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 9ad2a79b5..72defa7b7 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -17,6 +17,7 @@ from bs4 import BeautifulSoup from packageurl import PackageURL +from progress.bar import ChargingBar from univers.versions import AlpineLinuxVersion from vulnerabilities.importer import AdvisoryData @@ -41,18 +42,26 @@ def advisory_data(self) -> Iterable[AdvisoryData]: page_response_content = fetch_response(BASE_URL).content advisory_directory_links = fetch_advisory_directory_links(page_response_content) advisory_links = [] - for advisory_directory_link in advisory_directory_links: - advisory_directory_page = fetch_response(advisory_directory_link).content - advisory_links.extend( - fetch_advisory_links(advisory_directory_page, advisory_directory_link) - ) - for link in advisory_links: - record = fetch_response(link).json() - if not record["packages"]: - LOGGER.error(f'"packages" not found in {link!r}') - continue - yield from process_record(record=record, url=link) - + progress_for_package_fetch = ChargingBar("\tFetching Packages", max=len(advisory_directory_links)) + + try: + progress_for_package_fetch.start() + for advisory_directory_link in advisory_directory_links: + advisory_directory_page = fetch_response(advisory_directory_link).content + advisory_links.extend( + fetch_advisory_links(advisory_directory_page, advisory_directory_link) + ) + for link in advisory_links: + try: + record = fetch_response(link).json() + if not record["packages"]: + LOGGER.error(f'"packages" not found in {link!r}') + continue + yield from process_record(record=record, url=link) + finally: + progress_for_package_fetch.next() + finally: + progress_for_package_fetch.finish() def fetch_advisory_directory_links(page_response_content: str) -> List[str]: """ From 33c1533a26de90ac49a323d9a32d8643dc2ef6e8 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sat, 30 Dec 2023 01:23:46 +0530 Subject: [PATCH 10/62] Add Progress bar to OpensslImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/openssl.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/importers/openssl.py b/vulnerabilities/importers/openssl.py index ca69436c9..d84c3ec58 100644 --- a/vulnerabilities/importers/openssl.py +++ b/vulnerabilities/importers/openssl.py @@ -16,6 +16,7 @@ import requests from dateutil import parser as dateparser from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_range import OpensslVersionRange from univers.versions import OpensslVersion @@ -49,11 +50,20 @@ def advisory_data(self) -> Iterable[AdvisoryData]: def parse_vulnerabilities(xml_response) -> Iterable[AdvisoryData]: root = DET.fromstring(xml_response) - for xml_issue in root: - if xml_issue.tag == "issue": - advisory = to_advisory_data(xml_issue) - if advisory: - yield advisory + progress_bar_for_vulnerability_fetch = ChargingBar("\tFetching Vulnerabilities", max=len(root)) + progress_bar_for_vulnerability_fetch.start() + try: + for xml_issue in root: + try: + if xml_issue.tag == "issue": + advisory = to_advisory_data(xml_issue) + if advisory: + yield advisory + finally: + progress_bar_for_vulnerability_fetch.next() + finally: + progress_bar_for_vulnerability_fetch.finish() + def to_advisory_data(xml_issue) -> AdvisoryData: From 29b679c055e91cc326ab4b47de7f9a010dd913dc Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sat, 30 Dec 2023 05:01:15 +0530 Subject: [PATCH 11/62] Add Progress bar to DebianImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/debian.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py index 94057675f..5965f7aa7 100644 --- a/vulnerabilities/importers/debian.py +++ b/vulnerabilities/importers/debian.py @@ -15,6 +15,7 @@ import requests from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_range import DebianVersionRange from univers.versions import DebianVersion @@ -89,8 +90,16 @@ def get_response(self): def advisory_data(self) -> Iterable[AdvisoryData]: response = self.get_response() - for pkg_name, records in response.items(): - yield from self.parse(pkg_name, records) + progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(response.items())) + progress_bar_for_package_fetch.start() + try: + for pkg_name, records in response.items(): + try: + yield from self.parse(pkg_name, records) + finally: + progress_bar_for_package_fetch.next() + finally: + progress_bar_for_package_fetch.finish() def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryData]: for cve_id, record in records.items(): From cc2d29d33babd764cfe788d2dcf0db1879b21788 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sat, 30 Dec 2023 23:18:18 +0530 Subject: [PATCH 12/62] Add Progress bar to PostgreSQLImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/postgresql.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/importers/postgresql.py b/vulnerabilities/importers/postgresql.py index ee6de3976..5c841b888 100644 --- a/vulnerabilities/importers/postgresql.py +++ b/vulnerabilities/importers/postgresql.py @@ -12,6 +12,7 @@ import requests from bs4 import BeautifulSoup from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_range import GenericVersionRange from univers.versions import GenericVersion @@ -34,6 +35,7 @@ def advisory_data(self): known_urls = {self.root_url} visited_urls = set() data_by_url = {} + progress_bar_for_advisory_fetch = ChargingBar("\tFetching Advisories") while True: unvisited_urls = known_urls - visited_urls for url in unvisited_urls: @@ -45,9 +47,16 @@ def advisory_data(self): if known_urls == visited_urls: break - for url, data in data_by_url.items(): - yield from to_advisories(data) - + progress_bar_for_advisory_fetch.max = len(data_by_url) + progress_bar_for_advisory_fetch.start() + try: + for url, data in data_by_url.items(): + try: + yield from to_advisories(data) + finally: + progress_bar_for_advisory_fetch.next() + finally: + progress_bar_for_advisory_fetch.finish() def to_advisories(data): advisories = [] From ead8c1a0484f67485df9a7b5a264b269808a33ab Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 01:54:03 +0530 Subject: [PATCH 13/62] Add Progress bar to UbuntuImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/ubuntu.py | 39 ++++++++++++++++++----------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py index 646b40028..def58ca30 100644 --- a/vulnerabilities/importers/ubuntu.py +++ b/vulnerabilities/importers/ubuntu.py @@ -12,6 +12,7 @@ import xml.etree.ElementTree as ET import requests +from progress.bar import ChargingBar from vulnerabilities.importer import OvalImporter @@ -73,19 +74,27 @@ def __init__(self, *args, **kwargs): def _fetch(self): base_url = "https://people.canonical.com/~ubuntu-security/oval" releases = ["bionic", "trusty", "focal", "eoan", "xenial"] - for release in releases: - file_url = f"{base_url}/com.ubuntu.{release}.cve.oval.xml.bz2" # nopep8 - self.data_url = file_url - logger.info(f"Fetching Ubuntu Oval: {file_url}") - response = requests.get(file_url) - if response.status_code != requests.codes.ok: - logger.error( - f"Failed to fetch Ubuntu Oval: HTTP {response.status_code} : {file_url}" - ) - continue + progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(releases)) + progress_bar_for_package_fetch.start() + try: + for release in releases: + try: + file_url = f"{base_url}/com.ubuntu.{release}.cve.oval.xml.bz2" # nopep8 + self.data_url = file_url + logger.info(f"Fetching Ubuntu Oval: {file_url}") + response = requests.get(file_url) + if response.status_code != requests.codes.ok: + logger.error( + f"Failed to fetch Ubuntu Oval: HTTP {response.status_code} : {file_url}" + ) + continue - extracted = bz2.decompress(response.content) - yield ( - {"type": "deb", "namespace": "ubuntu"}, - ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))), - ) + extracted = bz2.decompress(response.content) + yield ( + {"type": "deb", "namespace": "ubuntu"}, + ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))), + ) + finally: + progress_bar_for_package_fetch.next() + finally: + progress_bar_for_package_fetch.finish() From 871bafc5b1ca4bcf68782de6c2e9f7741d0a235d Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 02:56:17 +0530 Subject: [PATCH 14/62] Add Progress bar to DebianOvalImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/debian_oval.py | 27 ++++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/vulnerabilities/importers/debian_oval.py b/vulnerabilities/importers/debian_oval.py index aa3d6917d..64fe609fd 100644 --- a/vulnerabilities/importers/debian_oval.py +++ b/vulnerabilities/importers/debian_oval.py @@ -12,6 +12,7 @@ import xml.etree.ElementTree as ET import requests +from progress.bar import ChargingBar from vulnerabilities.importer import OvalImporter @@ -65,12 +66,20 @@ def __init__(self, *args, **kwargs): def _fetch(self): releases = ["wheezy", "stretch", "jessie", "buster", "bullseye"] - for release in releases: - file_url = f"https://www.debian.org/security/oval/oval-definitions-{release}.xml.bz2" - self.data_url = file_url - resp = requests.get(file_url).content - extracted = bz2.decompress(resp) - yield ( - {"type": "deb", "namespace": "debian", "qualifiers": {"distro": release}}, - ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))), - ) + progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(releases)) + progress_bar_for_package_fetch.start() + try: + for release in releases: + try: + file_url = f"https://www.debian.org/security/oval/oval-definitions-{release}.xml.bz2" + self.data_url = file_url + resp = requests.get(file_url).content + extracted = bz2.decompress(resp) + yield ( + {"type": "deb", "namespace": "debian", "qualifiers": {"distro": release}}, + ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))), + ) + finally: + progress_bar_for_package_fetch.next() + finally: + progress_bar_for_package_fetch.finish() From b6559a351bc0cc3ec4bc698898c40a5a658e1de4 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 03:04:13 +0530 Subject: [PATCH 15/62] Add Progress bar to RetireDotnetImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/retiredotnet.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/importers/retiredotnet.py b/vulnerabilities/importers/retiredotnet.py index 0ed580ba1..35e01acfb 100644 --- a/vulnerabilities/importers/retiredotnet.py +++ b/vulnerabilities/importers/retiredotnet.py @@ -14,6 +14,7 @@ from typing import List from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_range import NugetVersionRange from univers.versions import NugetVersion @@ -31,16 +32,24 @@ class RetireDotnetImporter(Importer): importer_name = "RetireDotNet Importer" def advisory_data(self) -> Iterable[AdvisoryData]: + progress_bar_for_fetched_files: ChargingBar try: self.clone(repo_url=self.repo_url) base_path = Path(self.vcs_response.dest_dir) vuln = base_path / "Content" - for file in vuln.glob("*.json"): - advisory = self.process_file(file, base_path) - if advisory: - yield advisory + paths_for_vulnerabilities = list(vuln.glob("*.json")) + progress_bar_for_fetched_files = ChargingBar("\tFetching Vulnerabilities", max=len(paths_for_vulnerabilities)) + progress_bar_for_fetched_files.start() + for file in paths_for_vulnerabilities: + try: + advisory = self.process_file(file, base_path) + if advisory: + yield advisory + finally: + progress_bar_for_fetched_files.next() finally: + progress_bar_for_fetched_files.finish() if self.vcs_response: self.vcs_response.delete() From a035edd4b36716a30a60f0f1a19bf76547fa1bfb Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 03:30:46 +0530 Subject: [PATCH 16/62] Add Progress bar to ApacheHTTPDImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/apache_httpd.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index 10a99dd02..ae3ba4d5e 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -13,6 +13,7 @@ import requests from bs4 import BeautifulSoup from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_constraint import VersionConstraint from univers.version_range import ApacheVersionRange from univers.versions import SemverVersion @@ -37,9 +38,17 @@ class ApacheHTTPDImporter(Importer): def advisory_data(self): links = fetch_links(self.base_url) - for link in links: - data = requests.get(link).json() - yield self.to_advisory(data) + progress_bar_for_fetch_links = ChargingBar("\tFetching Vulnerabilitites", max=len(links)) + try: + progress_bar_for_fetch_links.start() + for link in links: + try: + data = requests.get(link).json() + yield self.to_advisory(data) + finally: + progress_bar_for_fetch_links.next() + finally: + progress_bar_for_fetch_links.finish() def to_advisory(self, data): alias = get_item(data, "CVE_data_meta", "ID") From a3f388e9da3ff1e6875d17ffe4d1bbf7a4b4cd69 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 03:45:57 +0530 Subject: [PATCH 17/62] Add Progress bar to MozillaImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/mozilla.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index 11667badc..dbd5ae32e 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -17,6 +17,7 @@ from bs4 import BeautifulSoup from markdown import markdown from packageurl import PackageURL +from progress.bar import ChargingBar from univers.versions import SemverVersion from vulnerabilities import severity_systems @@ -40,15 +41,22 @@ class MozillaImporter(Importer): importer_name = "Mozilla Importer" def advisory_data(self) -> Iterable[AdvisoryData]: + progress_bar_for_fetched_files: ChargingBar try: self.clone(self.repo_url) base_path = Path(self.vcs_response.dest_dir) vuln = base_path / "announce" paths = list(vuln.glob("**/*.yml")) + list(vuln.glob("**/*.md")) + progress_bar_for_fetched_files = ChargingBar("\tFetching Vulnerabilities", max=len(paths)) + progress_bar_for_fetched_files.start() for file_path in paths: - yield from to_advisories(file_path, base_path) + try: + yield from to_advisories(file_path, base_path) + finally: + progress_bar_for_fetched_files.next() finally: + progress_bar_for_fetched_files.finish() if self.vcs_response: self.vcs_response.delete() From 05a518fa8dd10671d04bb31e9978971ea89fb3ed Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 04:48:57 +0530 Subject: [PATCH 18/62] Add Progress bar to GentooImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/gentoo.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py index 2c91f7f2f..4faaebde8 100644 --- a/vulnerabilities/importers/gentoo.py +++ b/vulnerabilities/importers/gentoo.py @@ -14,6 +14,7 @@ from typing import Iterable from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_constraint import VersionConstraint from univers.version_range import EbuildVersionRange from univers.versions import GentooVersion @@ -34,12 +35,20 @@ class GentooImporter(Importer): importer_name = "Gentoo Importer" def advisory_data(self) -> Iterable[AdvisoryData]: + progress_bar_for_package_fetch: ChargingBar try: self.clone(repo_url=self.repo_url) base_path = Path(self.vcs_response.dest_dir) - for file_path in base_path.glob("**/*.xml"): - yield from self.process_file(file_path) + base_paths = list(base_path.glob("**/*.xml")) + progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(base_paths)) + progress_bar_for_package_fetch.start() + for file_path in base_paths: + try: + yield from self.process_file(file_path) + finally: + progress_bar_for_package_fetch.next() finally: + progress_bar_for_package_fetch.finish() if self.vcs_response: self.vcs_response.delete() From 7bc56092daaed7203f3a1f8a99f136d40f3ab122 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 06:51:46 +0530 Subject: [PATCH 19/62] Add Progress bar to IstioImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/istio.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 9341a76de..b1d326138 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -21,6 +21,7 @@ from dateutil import parser from django.db.models.query import QuerySet from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_constraint import VersionConstraint from univers.version_range import GitHubVersionRange from univers.version_range import GolangVersionRange @@ -45,19 +46,27 @@ class IstioImporter(Importer): importer_name = "Istio Importer" def advisory_data(self) -> Set[AdvisoryData]: + progress_bar_for_vuln_fetch: ChargingBar try: self.clone(repo_url=self.repo_url) base_path = Path(self.vcs_response.dest_dir) vuln = base_path / "content/en/news/security/" - for file in vuln.glob("**/*.md"): + file_paths_for_fetched_files = list(vuln.glob("**/*.md")) + progress_bar_for_vuln_fetch = ChargingBar("\tFetching Advisories", max=len(file_paths_for_fetched_files)) + progress_bar_for_vuln_fetch.start() + for file in file_paths_for_fetched_files: # Istio website has files with name starting with underscore, these contain metadata # required for rendering the website. We're not interested in these. # See also https://github.com/nexB/vulnerablecode/issues/563 - file = str(file) - if file.endswith("_index.md"): - continue - yield from self.process_file(file=file, base_path=base_path) + try: + file = str(file) + if file.endswith("_index.md"): + continue + yield from self.process_file(file=file, base_path=base_path) + finally: + progress_bar_for_vuln_fetch.next() finally: + progress_bar_for_vuln_fetch.finish() if self.vcs_response: self.vcs_response.delete() From bb56c7c4a9d0f5795e152e7517bf4e2ec45a4df4 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 08:23:36 +0530 Subject: [PATCH 20/62] Add Progress bar to ElixirSecurityImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/elixir_security.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index 4fd492a92..780a46673 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -12,6 +12,7 @@ from dateutil import parser as dateparser from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_constraint import VersionConstraint from univers.version_range import HexVersionRange @@ -31,13 +32,21 @@ class ElixirSecurityImporter(Importer): importer_name = "Elixir Security Importer" def advisory_data(self) -> Set[AdvisoryData]: + progress_bar_for_cve_fetch: ChargingBar try: self.clone(self.repo_url) base_path = Path(self.vcs_response.dest_dir) vuln = base_path / "packages" - for file in vuln.glob("**/*.yml"): - yield from self.process_file(file, base_path) + vuln_files = list(vuln.glob("**/*.yml")) + progress_bar_for_cve_fetch = ChargingBar("\tFetching CVEs", max=len(vuln_files)) + progress_bar_for_cve_fetch.start() + for file in vuln_files: + try: + yield from self.process_file(file, base_path) + finally: + progress_bar_for_cve_fetch.next() finally: + progress_bar_for_cve_fetch.finish() if self.vcs_response: self.vcs_response.delete() From be052f0e48a7bd5df799f05c02031cb40f3c825f Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 08:29:24 +0530 Subject: [PATCH 21/62] Add Progress bar to ApacheTomcatImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/apache_tomcat.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py index 50a02a0ec..f29036b1f 100644 --- a/vulnerabilities/importers/apache_tomcat.py +++ b/vulnerabilities/importers/apache_tomcat.py @@ -15,6 +15,7 @@ import requests from bs4 import BeautifulSoup from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_constraint import VersionConstraint from univers.version_range import ApacheVersionRange from univers.version_range import MavenVersionRange @@ -124,9 +125,17 @@ def fetch_advisory_pages(self): """ Yield the content of each HTML page containing version-related security data. """ - links = self.fetch_advisory_links("https://tomcat.apache.org/security") - for page_url in links: - yield page_url, requests.get(page_url).content + links = list(self.fetch_advisory_links("https://tomcat.apache.org/security")) + progress_bar_for_advisory_fetch = ChargingBar("\tFetching Advisories", max=len(links)) + progress_bar_for_advisory_fetch.start() + try: + for page_url in links: + try: + yield page_url, requests.get(page_url).content + finally: + progress_bar_for_advisory_fetch.next() + finally: + progress_bar_for_advisory_fetch.finish() def fetch_advisory_links(self, url): """ From 9d672fe8295aeef55127054009633a18244a7e28 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 08:34:46 +0530 Subject: [PATCH 22/62] Add Progress bar to XenImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/xen.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/xen.py b/vulnerabilities/importers/xen.py index 2b28a9771..cfd90289e 100644 --- a/vulnerabilities/importers/xen.py +++ b/vulnerabilities/importers/xen.py @@ -6,6 +6,7 @@ # See https://github.com/nexB/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +from progress.bar import ChargingBar from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer @@ -64,8 +65,16 @@ def advisory_data(self): if not data: return [] xsas = data[0]["xsas"] - for xsa in xsas: - yield from self.to_advisories(xsa) + progress_bar_for_advisory_fetch = ChargingBar("\tFetching Advisories", max=len(xsas)) + progress_bar_for_advisory_fetch.start() + try: + for xsa in xsas: + try: + yield from self.to_advisories(xsa) + finally: + progress_bar_for_advisory_fetch.next() + finally: + progress_bar_for_advisory_fetch.finish() def to_advisories(self, xsa): xsa_id = xsa.get("xsa") From 7cd57522bbfe2cd9b087e812eb921ec456b0959c Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 09:30:39 +0530 Subject: [PATCH 23/62] Add Progress bar to UbuntuUSNImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/ubuntu_usn.py | 45 +++++++++++++++---------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/vulnerabilities/importers/ubuntu_usn.py b/vulnerabilities/importers/ubuntu_usn.py index 6bef117f9..5c84f4788 100644 --- a/vulnerabilities/importers/ubuntu_usn.py +++ b/vulnerabilities/importers/ubuntu_usn.py @@ -11,6 +11,7 @@ import json import requests +from progress.bar import ChargingBar from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer @@ -69,25 +70,33 @@ def advisory_data(self): yield from self.to_advisories(usn_db=usn_db) def to_advisories(self, usn_db): - for usn in usn_db: - usn_data = usn_db[usn] - usn_reference = get_usn_reference(usn_data.get("id")) - usn_references = [] - if usn_reference: - usn_references = [usn_reference] - for cve in usn_data.get("cves", []): - # The db sometimes contains entries like - # {'cves': ['python-pgsql vulnerabilities', 'CVE-2006-2313', 'CVE-2006-2314']} - # This `if` filters entries like 'python-pgsql vulnerabilities' - if not is_cve(cve): - continue + progress_bar_for_advisory_fetch = ChargingBar("\tFetching Advisories", max=len(usn_db)) + try: + progress_bar_for_advisory_fetch.start() + for usn in usn_db: + try: + usn_data = usn_db[usn] + usn_reference = get_usn_reference(usn_data.get("id")) + usn_references = [] + if usn_reference: + usn_references = [usn_reference] + for cve in usn_data.get("cves", []): + # The db sometimes contains entries like + # {'cves': ['python-pgsql vulnerabilities', 'CVE-2006-2313', 'CVE-2006-2314']} + # This `if` filters entries like 'python-pgsql vulnerabilities' + if not is_cve(cve): + continue - yield AdvisoryData( - aliases=[cve], - summary="", - references=usn_references, - url=usn_reference.url or self.db_url, - ) + yield AdvisoryData( + aliases=[cve], + summary="", + references=usn_references, + url=usn_reference.url or self.db_url, + ) + finally: + progress_bar_for_advisory_fetch.next() + finally: + progress_bar_for_advisory_fetch.finish() def get_usn_reference(usn_id): From 97074ea27ed9d57f200e25cc754ea80d31eb0785 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 09:42:00 +0530 Subject: [PATCH 24/62] Add Progress bar to FireyeImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/fireeye.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/fireeye.py b/vulnerabilities/importers/fireeye.py index 814af7248..9af5e7751 100644 --- a/vulnerabilities/importers/fireeye.py +++ b/vulnerabilities/importers/fireeye.py @@ -12,6 +12,8 @@ from typing import Iterable from typing import List +from progress.bar import ChargingBar + from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference @@ -34,12 +36,16 @@ class FireyeImporter(Importer): importer_name = "FireEye Importer" def advisory_data(self) -> Iterable[AdvisoryData]: + progress_bar_for_advisory_fetch: ChargingBar try: self.vcs_response = self.clone(repo_url=self.repo_url) base_path = Path(self.vcs_response.dest_dir) - files = filter( + files = list(filter( lambda p: p.suffix in [".md", ".MD"], Path(self.vcs_response.dest_dir).glob("**/*") - ) + )) + progress_bar_for_advisory_fetch = ChargingBar("\tFetching Advisories", max=len(files)) + progress_bar_for_advisory_fetch.start() + for file in files: if Path(file).stem == "README": continue @@ -48,7 +54,10 @@ def advisory_data(self) -> Iterable[AdvisoryData]: yield parse_advisory_data(raw_data=f.read(), file=file, base_path=base_path) except UnicodeError: logger.error(f"Invalid file {file}") + finally: + progress_bar_for_advisory_fetch.next() finally: + progress_bar_for_advisory_fetch.finish() if self.vcs_response: self.vcs_response.delete() @@ -64,6 +73,7 @@ def parse_advisory_data(raw_data, file, base_path) -> AdvisoryData: ) raw_data = raw_data.replace("\n\n", "\n") md_list = raw_data.split("\n") + print(md_list) md_dict = md_list_to_dict(md_list) database_id = md_list[0][1::] From e61155787c07d179c1feec9d0b298a691d052ebc Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 09:57:48 +0530 Subject: [PATCH 25/62] Add Progress bar to ApacheKafkaImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/apache_kafka.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vulnerabilities/importers/apache_kafka.py b/vulnerabilities/importers/apache_kafka.py index 89de85219..4fa6ab5c3 100644 --- a/vulnerabilities/importers/apache_kafka.py +++ b/vulnerabilities/importers/apache_kafka.py @@ -15,6 +15,7 @@ from bs4 import BeautifulSoup from dateutil.parser import parse from packageurl import PackageURL +from progress.bar import ChargingBar from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage @@ -113,6 +114,8 @@ def to_advisory(self, advisory_page): advisory_page = BeautifulSoup(advisory_page, features="lxml") cve_section_beginnings = advisory_page.find_all("h2") + progress_bar_for_cve_fetch = ChargingBar("\tFetching CVEs", max=len(cve_section_beginnings)) + progress_bar_for_cve_fetch.start() for cve_section_beginning in cve_section_beginnings: # This sometimes includes text that follows the CVE on the same line -- sometimes there is a carriage return, sometimes there is not # cve_id = cve_section_beginning.text.split("\n")[0] @@ -195,5 +198,8 @@ def to_advisory(self, advisory_page): url=f"{self.ASF_PAGE_URL}#{cve_id}", ) ) + progress_bar_for_cve_fetch.next() + + progress_bar_for_cve_fetch.finish() return advisories From aa7fad85ec9686fa50726b820ef2c955bc15a290 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 10:05:29 +0530 Subject: [PATCH 26/62] Add Progress bar to OSSFuzzImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/oss_fuzz.py | 31 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/vulnerabilities/importers/oss_fuzz.py b/vulnerabilities/importers/oss_fuzz.py index e86df5ce8..2c7f7927f 100644 --- a/vulnerabilities/importers/oss_fuzz.py +++ b/vulnerabilities/importers/oss_fuzz.py @@ -11,6 +11,7 @@ from typing import Iterable import saneyaml +from progress.bar import ChargingBar from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer @@ -27,21 +28,29 @@ class OSSFuzzImporter(Importer): importer_name = "OSS Fuzz Importer" def advisory_data(self) -> Iterable[AdvisoryData]: + progress_bar_for_cve_fetch = ChargingBar("\tFetching CVEs") try: self.clone(repo_url=self.url) base_path = Path(self.vcs_response.dest_dir) path = base_path / "vulns" - for file in path.glob("**/*.yaml"): - with open(file) as f: - yaml_data = saneyaml.load(f.read()) - advisory_url = get_advisory_url( - file=file, - base_path=base_path, - url="https://github.com/pypa/advisory-database/blob/main/", - ) - yield parse_advisory_data( - yaml_data, supported_ecosystem="oss-fuzz", advisory_url=advisory_url - ) + files = list(path.glob("**/*.yaml")) + progress_bar_for_cve_fetch.max = len(files) + progress_bar_for_cve_fetch.start() + for file in files: + try: + with open(file) as f: + yaml_data = saneyaml.load(f.read()) + advisory_url = get_advisory_url( + file=file, + base_path=base_path, + url="https://github.com/pypa/advisory-database/blob/main/", + ) + yield parse_advisory_data( + yaml_data, supported_ecosystem="oss-fuzz", advisory_url=advisory_url + ) + finally: + progress_bar_for_cve_fetch.next() finally: + progress_bar_for_cve_fetch.next() if self.vcs_response: self.vcs_response.delete() From ac649a9783eb2950756a08a38132fadb6491d508 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 10:14:13 +0530 Subject: [PATCH 27/62] Add Progress bar to SUSESeverityScoreImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/suse_scores.py | 56 ++++++++++++++---------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/vulnerabilities/importers/suse_scores.py b/vulnerabilities/importers/suse_scores.py index f43e69576..a0a2b9692 100644 --- a/vulnerabilities/importers/suse_scores.py +++ b/vulnerabilities/importers/suse_scores.py @@ -9,6 +9,8 @@ from typing import Iterable +from progress.bar import ChargingBar + from vulnerabilities import severity_systems from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer @@ -37,28 +39,36 @@ def to_advisory(self, score_data): "3.1": severity_systems.CVSSV31, } - for cve_id in score_data or []: - severities = [] - for cvss_score in score_data[cve_id].get("cvss") or []: - cvss_version = cvss_score.get("version") or "" - scoring_system = systems_by_version.get(cvss_version) - if not scoring_system: - continue - base_score = str(cvss_score.get("score") or "") - vector = str(cvss_score.get("vector") or "") - score = VulnerabilitySeverity( - system=scoring_system, - value=base_score, - scoring_elements=vector, - ) - severities.append(score) + progress_bar_for_cve_fetch = ChargingBar("\tFetching CVEs", max=len(score_data or [])) + progress_bar_for_cve_fetch.start() + try: + for cve_id in score_data or []: + try: + severities = [] + for cvss_score in score_data[cve_id].get("cvss") or []: + cvss_version = cvss_score.get("version") or "" + scoring_system = systems_by_version.get(cvss_version) + if not scoring_system: + continue + base_score = str(cvss_score.get("score") or "") + vector = str(cvss_score.get("vector") or "") + score = VulnerabilitySeverity( + system=scoring_system, + value=base_score, + scoring_elements=vector, + ) + severities.append(score) - if not is_cve(cve_id): - continue + if not is_cve(cve_id): + continue - yield AdvisoryData( - aliases=[cve_id], - summary="", - references=[Reference(url=URL, severities=severities)], - url=URL, - ) + yield AdvisoryData( + aliases=[cve_id], + summary="", + references=[Reference(url=URL, severities=severities)], + url=URL, + ) + finally: + progress_bar_for_cve_fetch.next() + finally: + progress_bar_for_cve_fetch.finish() From b8224fec163bba3eaf5d83c698877f3885bb358e Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 10:20:24 +0530 Subject: [PATCH 28/62] Add Progress bar to ArchlinuxImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/archlinux.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py index 2e9ef6a87..342067a8a 100644 --- a/vulnerabilities/importers/archlinux.py +++ b/vulnerabilities/importers/archlinux.py @@ -12,6 +12,7 @@ from typing import Mapping from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_range import ArchLinuxVersionRange from univers.versions import ArchLinuxVersion @@ -35,8 +36,17 @@ def fetch(self) -> Iterable[Mapping]: return response.json() def advisory_data(self) -> Iterable[AdvisoryData]: - for record in self.fetch(): - yield from self.parse_advisory(record) + records = self.fetch() + progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(records or [])) + progress_bar_for_package_fetch.start() + try: + for record in records: + try: + yield from self.parse_advisory(record) + finally: + progress_bar_for_package_fetch.next() + finally: + progress_bar_for_package_fetch.finish() def parse_advisory(self, record) -> List[AdvisoryData]: advisories = [] From 0c448275858474b67eebe4648a8acbeff47e43ff Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 10:49:54 +0530 Subject: [PATCH 29/62] Add Progress bar to ProjectKBMSRImporter Signed-off-by: Harsh Mishra --- .../importers/project_kb_msr2019.py | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/vulnerabilities/importers/project_kb_msr2019.py b/vulnerabilities/importers/project_kb_msr2019.py index e099d3f36..71a3c4a7e 100644 --- a/vulnerabilities/importers/project_kb_msr2019.py +++ b/vulnerabilities/importers/project_kb_msr2019.py @@ -6,6 +6,7 @@ # See https://github.com/nexB/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +from progress.bar import ChargingBar from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer @@ -30,17 +31,32 @@ def advisory_data(self): def to_advisories(self, csv_reader): # Project KB MSR csv file has no header row - for row in csv_reader: - vuln_id, proj_home, fix_commit, _ = row - commit_link = proj_home + "/commit/" + fix_commit - - if not is_cve(vuln_id): - continue - - reference = Reference(url=commit_link) - yield AdvisoryData( - aliases=[vuln_id], - summary="", - references=[reference], - url=self.url, - ) + + # This refactoring can cause excessive memory usage. + # But there was no better way to do with current library. + # Might consider upgrading to TQDM in future + rows = list(csv_reader) + progress_bar_for_cve_fetch = ChargingBar("\tFetching CVEs", max=len(rows)) + progress_bar_for_cve_fetch.start() + counter = 0 + try: + for row in rows: + try: + vuln_id, proj_home, fix_commit, _ = row + commit_link = proj_home + "/commit/" + fix_commit + + if not is_cve(vuln_id): + continue + + reference = Reference(url=commit_link) + yield AdvisoryData( + aliases=[vuln_id], + summary="", + references=[reference], + url=self.url, + ) + finally: + progress_bar_for_cve_fetch.next() + finally: + progress_bar_for_cve_fetch.finish() + print("Finish") From c7146a32f58375aa35783e4a84c02f94d3fc7220 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Sun, 31 Dec 2023 10:58:05 +0530 Subject: [PATCH 30/62] Add Progress bar to RedhatImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/redhat.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index a2cc1940b..bbae46f75 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -15,6 +15,7 @@ import requests from packageurl import PackageURL +from progress.bar import ChargingBar from univers.version_range import RpmVersionRange from vulnerabilities import severity_systems @@ -68,9 +69,18 @@ class RedhatImporter(Importer): importer_name = "RedHat Importer" def advisory_data(self) -> Iterable[AdvisoryData]: + page_no = 1 for redhat_cves in fetch_cves(): - for redhat_cve in redhat_cves: - yield to_advisory(redhat_cve) + progress_bar_for_cve_fetch = ChargingBar(f"\tFetching CVE Set-{page_no}", max=len(redhat_cves)) + try: + for redhat_cve in redhat_cves: + try: + yield to_advisory(redhat_cve) + finally: + progress_bar_for_cve_fetch.next() + finally: + progress_bar_for_cve_fetch.finish() + page_no+=1 def to_advisory(advisory_data): From 895a8b68925be7ef169d8054cb789fb0ec2a6829 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 2 Jan 2024 16:48:57 +0530 Subject: [PATCH 31/62] Fix Progress bar for RedhatImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/redhat.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index bbae46f75..dfda9c40a 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -72,15 +72,11 @@ def advisory_data(self) -> Iterable[AdvisoryData]: page_no = 1 for redhat_cves in fetch_cves(): progress_bar_for_cve_fetch = ChargingBar(f"\tFetching CVE Set-{page_no}", max=len(redhat_cves)) - try: - for redhat_cve in redhat_cves: - try: - yield to_advisory(redhat_cve) - finally: - progress_bar_for_cve_fetch.next() - finally: - progress_bar_for_cve_fetch.finish() - page_no+=1 + for redhat_cve in redhat_cves: + yield to_advisory(redhat_cve) + progress_bar_for_cve_fetch.next() + progress_bar_for_cve_fetch.finish() + page_no+=1 def to_advisory(advisory_data): From f988170c4cd0ec5dea3cdc83e1f54b3343186ce3 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 2 Jan 2024 16:52:12 +0530 Subject: [PATCH 32/62] Fix Progress bar for OSSFuzzImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/oss_fuzz.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/vulnerabilities/importers/oss_fuzz.py b/vulnerabilities/importers/oss_fuzz.py index 2c7f7927f..dc5006355 100644 --- a/vulnerabilities/importers/oss_fuzz.py +++ b/vulnerabilities/importers/oss_fuzz.py @@ -37,20 +37,18 @@ def advisory_data(self) -> Iterable[AdvisoryData]: progress_bar_for_cve_fetch.max = len(files) progress_bar_for_cve_fetch.start() for file in files: - try: - with open(file) as f: - yaml_data = saneyaml.load(f.read()) - advisory_url = get_advisory_url( - file=file, - base_path=base_path, - url="https://github.com/pypa/advisory-database/blob/main/", - ) - yield parse_advisory_data( - yaml_data, supported_ecosystem="oss-fuzz", advisory_url=advisory_url - ) - finally: - progress_bar_for_cve_fetch.next() + with open(file) as f: + yaml_data = saneyaml.load(f.read()) + advisory_url = get_advisory_url( + file=file, + base_path=base_path, + url="https://github.com/pypa/advisory-database/blob/main/", + ) + yield parse_advisory_data( + yaml_data, supported_ecosystem="oss-fuzz", advisory_url=advisory_url + ) + progress_bar_for_cve_fetch.next() finally: - progress_bar_for_cve_fetch.next() + progress_bar_for_cve_fetch.finish() if self.vcs_response: self.vcs_response.delete() From 0841f557aedacea5b14c1b9154953adca4da897e Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 2 Jan 2024 17:23:31 +0530 Subject: [PATCH 33/62] Fix Progress bar for UbuntuUSNImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/ubuntu_usn.py | 47 ++++++++++++------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/vulnerabilities/importers/ubuntu_usn.py b/vulnerabilities/importers/ubuntu_usn.py index 5c84f4788..4672794d4 100644 --- a/vulnerabilities/importers/ubuntu_usn.py +++ b/vulnerabilities/importers/ubuntu_usn.py @@ -71,32 +71,29 @@ def advisory_data(self): def to_advisories(self, usn_db): progress_bar_for_advisory_fetch = ChargingBar("\tFetching Advisories", max=len(usn_db)) - try: - progress_bar_for_advisory_fetch.start() - for usn in usn_db: - try: - usn_data = usn_db[usn] - usn_reference = get_usn_reference(usn_data.get("id")) - usn_references = [] - if usn_reference: - usn_references = [usn_reference] - for cve in usn_data.get("cves", []): - # The db sometimes contains entries like - # {'cves': ['python-pgsql vulnerabilities', 'CVE-2006-2313', 'CVE-2006-2314']} - # This `if` filters entries like 'python-pgsql vulnerabilities' - if not is_cve(cve): - continue + progress_bar_for_advisory_fetch.start() + for usn in usn_db: + usn_data = usn_db[usn] + usn_reference = get_usn_reference(usn_data.get("id")) + usn_references = [] + if usn_reference: + usn_references = [usn_reference] + for cve in usn_data.get("cves", []): + # The db sometimes contains entries like + # {'cves': ['python-pgsql vulnerabilities', 'CVE-2006-2313', 'CVE-2006-2314']} + # This `if` filters entries like 'python-pgsql vulnerabilities' + if not is_cve(cve): + continue - yield AdvisoryData( - aliases=[cve], - summary="", - references=usn_references, - url=usn_reference.url or self.db_url, - ) - finally: - progress_bar_for_advisory_fetch.next() - finally: - progress_bar_for_advisory_fetch.finish() + yield AdvisoryData( + aliases=[cve], + summary="", + references=usn_references, + url=usn_reference.url or self.db_url, + ) + progress_bar_for_advisory_fetch.next() + + progress_bar_for_advisory_fetch.finish() def get_usn_reference(usn_id): From d5e46f8848e50a9743a31dcb6fcd517fca0c337d Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 2 Jan 2024 17:23:56 +0530 Subject: [PATCH 34/62] Fix Progress bar for XenImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/xen.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/importers/xen.py b/vulnerabilities/importers/xen.py index cfd90289e..884230ff4 100644 --- a/vulnerabilities/importers/xen.py +++ b/vulnerabilities/importers/xen.py @@ -67,14 +67,12 @@ def advisory_data(self): xsas = data[0]["xsas"] progress_bar_for_advisory_fetch = ChargingBar("\tFetching Advisories", max=len(xsas)) progress_bar_for_advisory_fetch.start() - try: - for xsa in xsas: - try: - yield from self.to_advisories(xsa) - finally: - progress_bar_for_advisory_fetch.next() - finally: - progress_bar_for_advisory_fetch.finish() + + for xsa in xsas: + yield from self.to_advisories(xsa) + progress_bar_for_advisory_fetch.next() + + progress_bar_for_advisory_fetch.finish() def to_advisories(self, xsa): xsa_id = xsa.get("xsa") From dd8da4c19518c11b2193b6835b267b726c0ba0e1 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 2 Jan 2024 17:26:24 +0530 Subject: [PATCH 35/62] Fix Progress bar for ApacheTomcatImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/apache_tomcat.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py index f29036b1f..a0344169c 100644 --- a/vulnerabilities/importers/apache_tomcat.py +++ b/vulnerabilities/importers/apache_tomcat.py @@ -128,14 +128,11 @@ def fetch_advisory_pages(self): links = list(self.fetch_advisory_links("https://tomcat.apache.org/security")) progress_bar_for_advisory_fetch = ChargingBar("\tFetching Advisories", max=len(links)) progress_bar_for_advisory_fetch.start() - try: - for page_url in links: - try: - yield page_url, requests.get(page_url).content - finally: - progress_bar_for_advisory_fetch.next() - finally: - progress_bar_for_advisory_fetch.finish() + for page_url in links: + yield page_url, requests.get(page_url).content + progress_bar_for_advisory_fetch.next() + + progress_bar_for_advisory_fetch.finish() def fetch_advisory_links(self, url): """ From 8cd8902f117752e62f4025ec345e25b5af3b0a01 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 2 Jan 2024 17:30:12 +0530 Subject: [PATCH 36/62] Fix Progress bar for ElixirSecurityImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/elixir_security.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index 780a46673..7939f5739 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -41,10 +41,8 @@ def advisory_data(self) -> Set[AdvisoryData]: progress_bar_for_cve_fetch = ChargingBar("\tFetching CVEs", max=len(vuln_files)) progress_bar_for_cve_fetch.start() for file in vuln_files: - try: - yield from self.process_file(file, base_path) - finally: - progress_bar_for_cve_fetch.next() + yield from self.process_file(file, base_path) + progress_bar_for_cve_fetch.next() finally: progress_bar_for_cve_fetch.finish() if self.vcs_response: From dc3e73b407f414bc53ff7433d9fe826fa5b315d0 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 05:26:25 +0530 Subject: [PATCH 37/62] Fix Readability in RedhatImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/redhat.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index dfda9c40a..48e675bc7 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -71,12 +71,14 @@ class RedhatImporter(Importer): def advisory_data(self) -> Iterable[AdvisoryData]: page_no = 1 for redhat_cves in fetch_cves(): - progress_bar_for_cve_fetch = ChargingBar(f"\tFetching CVE Set-{page_no}", max=len(redhat_cves)) + progress_bar_for_cve_fetch = ChargingBar( + f"\tFetching CVE Set-{page_no}", max=len(redhat_cves) + ) for redhat_cve in redhat_cves: yield to_advisory(redhat_cve) progress_bar_for_cve_fetch.next() progress_bar_for_cve_fetch.finish() - page_no+=1 + page_no += 1 def to_advisory(advisory_data): From 7495917d7a84a166dcb7e2d063539e2834a8e8e0 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 05:32:10 +0530 Subject: [PATCH 38/62] Fix Progress Bar in SUSESeverityScoreImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/suse_scores.py | 56 +++++++++++------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/vulnerabilities/importers/suse_scores.py b/vulnerabilities/importers/suse_scores.py index a0a2b9692..9f9eccbd7 100644 --- a/vulnerabilities/importers/suse_scores.py +++ b/vulnerabilities/importers/suse_scores.py @@ -38,37 +38,31 @@ def to_advisory(self, score_data): "3": severity_systems.CVSSV3, "3.1": severity_systems.CVSSV31, } - progress_bar_for_cve_fetch = ChargingBar("\tFetching CVEs", max=len(score_data or [])) progress_bar_for_cve_fetch.start() - try: - for cve_id in score_data or []: - try: - severities = [] - for cvss_score in score_data[cve_id].get("cvss") or []: - cvss_version = cvss_score.get("version") or "" - scoring_system = systems_by_version.get(cvss_version) - if not scoring_system: - continue - base_score = str(cvss_score.get("score") or "") - vector = str(cvss_score.get("vector") or "") - score = VulnerabilitySeverity( - system=scoring_system, - value=base_score, - scoring_elements=vector, - ) - severities.append(score) - - if not is_cve(cve_id): - continue + for cve_id in score_data or []: + severities = [] + for cvss_score in score_data[cve_id].get("cvss") or []: + cvss_version = cvss_score.get("version") or "" + scoring_system = systems_by_version.get(cvss_version) + if not scoring_system: + continue + base_score = str(cvss_score.get("score") or "") + vector = str(cvss_score.get("vector") or "") + score = VulnerabilitySeverity( + system=scoring_system, + value=base_score, + scoring_elements=vector, + ) + severities.append(score) + if not is_cve(cve_id): + continue - yield AdvisoryData( - aliases=[cve_id], - summary="", - references=[Reference(url=URL, severities=severities)], - url=URL, - ) - finally: - progress_bar_for_cve_fetch.next() - finally: - progress_bar_for_cve_fetch.finish() + yield AdvisoryData( + aliases=[cve_id], + summary="", + references=[Reference(url=URL, severities=severities)], + url=URL, + ) + progress_bar_for_cve_fetch.next() + progress_bar_for_cve_fetch.finish() From f4715f3eb1375cbf6b9543da2bc99f065f1ab09d Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 05:34:55 +0530 Subject: [PATCH 39/62] Fix Progress Bar in ProjectKBMSRImporter Signed-off-by: Harsh Mishra --- .../importers/project_kb_msr2019.py | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/vulnerabilities/importers/project_kb_msr2019.py b/vulnerabilities/importers/project_kb_msr2019.py index 71a3c4a7e..5f7041ddf 100644 --- a/vulnerabilities/importers/project_kb_msr2019.py +++ b/vulnerabilities/importers/project_kb_msr2019.py @@ -38,25 +38,17 @@ def to_advisories(self, csv_reader): rows = list(csv_reader) progress_bar_for_cve_fetch = ChargingBar("\tFetching CVEs", max=len(rows)) progress_bar_for_cve_fetch.start() - counter = 0 - try: - for row in rows: - try: - vuln_id, proj_home, fix_commit, _ = row - commit_link = proj_home + "/commit/" + fix_commit - - if not is_cve(vuln_id): - continue - - reference = Reference(url=commit_link) - yield AdvisoryData( - aliases=[vuln_id], - summary="", - references=[reference], - url=self.url, - ) - finally: - progress_bar_for_cve_fetch.next() - finally: - progress_bar_for_cve_fetch.finish() - print("Finish") + for row in rows: + vuln_id, proj_home, fix_commit, _ = row + commit_link = proj_home + "/commit/" + fix_commit + if not is_cve(vuln_id): + continue + reference = Reference(url=commit_link) + yield AdvisoryData( + aliases=[vuln_id], + summary="", + references=[reference], + url=self.url, + ) + progress_bar_for_cve_fetch.next() + progress_bar_for_cve_fetch.finish() From 69c86747ad96cfe48c60a1d4145a0cd0de20b2b8 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 05:44:38 +0530 Subject: [PATCH 40/62] Fix Progress Bar in IstioImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/istio.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index b1d326138..5dad1a255 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -52,19 +52,19 @@ def advisory_data(self) -> Set[AdvisoryData]: base_path = Path(self.vcs_response.dest_dir) vuln = base_path / "content/en/news/security/" file_paths_for_fetched_files = list(vuln.glob("**/*.md")) - progress_bar_for_vuln_fetch = ChargingBar("\tFetching Advisories", max=len(file_paths_for_fetched_files)) + progress_bar_for_vuln_fetch = ChargingBar( + "\tFetching Advisories", max=len(file_paths_for_fetched_files) + ) progress_bar_for_vuln_fetch.start() for file in file_paths_for_fetched_files: # Istio website has files with name starting with underscore, these contain metadata # required for rendering the website. We're not interested in these. # See also https://github.com/nexB/vulnerablecode/issues/563 - try: - file = str(file) - if file.endswith("_index.md"): - continue - yield from self.process_file(file=file, base_path=base_path) - finally: - progress_bar_for_vuln_fetch.next() + file = str(file) + if file.endswith("_index.md"): + continue + yield from self.process_file(file=file, base_path=base_path) + progress_bar_for_vuln_fetch.next() finally: progress_bar_for_vuln_fetch.finish() if self.vcs_response: From 331cdb77f33c504bdb2fbc24e8faf087a3c8b379 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 05:54:19 +0530 Subject: [PATCH 41/62] Fix Progress Bar in GentooImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/gentoo.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py index 4faaebde8..f51998575 100644 --- a/vulnerabilities/importers/gentoo.py +++ b/vulnerabilities/importers/gentoo.py @@ -43,10 +43,8 @@ def advisory_data(self) -> Iterable[AdvisoryData]: progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(base_paths)) progress_bar_for_package_fetch.start() for file_path in base_paths: - try: - yield from self.process_file(file_path) - finally: - progress_bar_for_package_fetch.next() + yield from self.process_file(file_path) + progress_bar_for_package_fetch.next() finally: progress_bar_for_package_fetch.finish() if self.vcs_response: From 8b86827687b1326dde693ee406b31d90db3c40cb Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 05:58:31 +0530 Subject: [PATCH 42/62] Fix Progress Bar in MozillaImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/mozilla.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index dbd5ae32e..d03437776 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -48,13 +48,13 @@ def advisory_data(self) -> Iterable[AdvisoryData]: vuln = base_path / "announce" paths = list(vuln.glob("**/*.yml")) + list(vuln.glob("**/*.md")) - progress_bar_for_fetched_files = ChargingBar("\tFetching Vulnerabilities", max=len(paths)) + progress_bar_for_fetched_files = ChargingBar( + "\tFetching Vulnerabilities", max=len(paths) + ) progress_bar_for_fetched_files.start() for file_path in paths: - try: - yield from to_advisories(file_path, base_path) - finally: - progress_bar_for_fetched_files.next() + yield from to_advisories(file_path, base_path) + progress_bar_for_fetched_files.next() finally: progress_bar_for_fetched_files.finish() if self.vcs_response: From bdbf439890d4e4ec7969754617aed4c8893f9ede Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:03:49 +0530 Subject: [PATCH 43/62] Fix Progress Bar in ApacheHTTPDImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/apache_httpd.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index ae3ba4d5e..e61301d29 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -39,16 +39,12 @@ class ApacheHTTPDImporter(Importer): def advisory_data(self): links = fetch_links(self.base_url) progress_bar_for_fetch_links = ChargingBar("\tFetching Vulnerabilitites", max=len(links)) - try: - progress_bar_for_fetch_links.start() - for link in links: - try: - data = requests.get(link).json() - yield self.to_advisory(data) - finally: - progress_bar_for_fetch_links.next() - finally: - progress_bar_for_fetch_links.finish() + progress_bar_for_fetch_links.start() + for link in links: + data = requests.get(link).json() + yield self.to_advisory(data) + progress_bar_for_fetch_links.next() + progress_bar_for_fetch_links.finish() def to_advisory(self, data): alias = get_item(data, "CVE_data_meta", "ID") From 77fcbe7162f92a035bdc5dc1b0e8d1946fafe571 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:06:42 +0530 Subject: [PATCH 44/62] Fix Progress Bar in RetireDotnetImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/retiredotnet.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/importers/retiredotnet.py b/vulnerabilities/importers/retiredotnet.py index 35e01acfb..1c67ec38d 100644 --- a/vulnerabilities/importers/retiredotnet.py +++ b/vulnerabilities/importers/retiredotnet.py @@ -36,18 +36,17 @@ def advisory_data(self) -> Iterable[AdvisoryData]: try: self.clone(repo_url=self.repo_url) base_path = Path(self.vcs_response.dest_dir) - vuln = base_path / "Content" paths_for_vulnerabilities = list(vuln.glob("*.json")) - progress_bar_for_fetched_files = ChargingBar("\tFetching Vulnerabilities", max=len(paths_for_vulnerabilities)) + progress_bar_for_fetched_files = ChargingBar( + "\tFetching Vulnerabilities", max=len(paths_for_vulnerabilities) + ) progress_bar_for_fetched_files.start() for file in paths_for_vulnerabilities: - try: - advisory = self.process_file(file, base_path) - if advisory: - yield advisory - finally: - progress_bar_for_fetched_files.next() + advisory = self.process_file(file, base_path) + if advisory: + yield advisory + progress_bar_for_fetched_files.next() finally: progress_bar_for_fetched_files.finish() if self.vcs_response: From d5128890954a6db55465879c06bee46bdf0af8f9 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:10:25 +0530 Subject: [PATCH 45/62] Fix Progress Bar in DebianOvalImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/debian_oval.py | 26 ++++++++++-------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/vulnerabilities/importers/debian_oval.py b/vulnerabilities/importers/debian_oval.py index 64fe609fd..8d3d82eab 100644 --- a/vulnerabilities/importers/debian_oval.py +++ b/vulnerabilities/importers/debian_oval.py @@ -68,18 +68,14 @@ def _fetch(self): releases = ["wheezy", "stretch", "jessie", "buster", "bullseye"] progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(releases)) progress_bar_for_package_fetch.start() - try: - for release in releases: - try: - file_url = f"https://www.debian.org/security/oval/oval-definitions-{release}.xml.bz2" - self.data_url = file_url - resp = requests.get(file_url).content - extracted = bz2.decompress(resp) - yield ( - {"type": "deb", "namespace": "debian", "qualifiers": {"distro": release}}, - ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))), - ) - finally: - progress_bar_for_package_fetch.next() - finally: - progress_bar_for_package_fetch.finish() + for release in releases: + file_url = f"https://www.debian.org/security/oval/oval-definitions-{release}.xml.bz2" + self.data_url = file_url + resp = requests.get(file_url).content + extracted = bz2.decompress(resp) + yield ( + {"type": "deb", "namespace": "debian", "qualifiers": {"distro": release}}, + ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))), + ) + progress_bar_for_package_fetch.next() + progress_bar_for_package_fetch.finish() From ced152afd61b5dfb94b1a010ef18b8a0eb635ee9 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:16:42 +0530 Subject: [PATCH 46/62] Fix Progress Bar in UbuntuImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/ubuntu.py | 39 +++++++++++++---------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py index def58ca30..54d64ec88 100644 --- a/vulnerabilities/importers/ubuntu.py +++ b/vulnerabilities/importers/ubuntu.py @@ -76,25 +76,20 @@ def _fetch(self): releases = ["bionic", "trusty", "focal", "eoan", "xenial"] progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(releases)) progress_bar_for_package_fetch.start() - try: - for release in releases: - try: - file_url = f"{base_url}/com.ubuntu.{release}.cve.oval.xml.bz2" # nopep8 - self.data_url = file_url - logger.info(f"Fetching Ubuntu Oval: {file_url}") - response = requests.get(file_url) - if response.status_code != requests.codes.ok: - logger.error( - f"Failed to fetch Ubuntu Oval: HTTP {response.status_code} : {file_url}" - ) - continue - - extracted = bz2.decompress(response.content) - yield ( - {"type": "deb", "namespace": "ubuntu"}, - ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))), - ) - finally: - progress_bar_for_package_fetch.next() - finally: - progress_bar_for_package_fetch.finish() + for release in releases: + file_url = f"{base_url}/com.ubuntu.{release}.cve.oval.xml.bz2" # nopep8 + self.data_url = file_url + logger.info(f"Fetching Ubuntu Oval: {file_url}") + response = requests.get(file_url) + if response.status_code != requests.codes.ok: + logger.error( + f"Failed to fetch Ubuntu Oval: HTTP {response.status_code} : {file_url}" + ) + continue + extracted = bz2.decompress(response.content) + yield ( + {"type": "deb", "namespace": "ubuntu"}, + ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))), + ) + progress_bar_for_package_fetch.next() + progress_bar_for_package_fetch.finish() From b94542cd0ef0807ae396e219bc949cacb9ac05d5 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:21:08 +0530 Subject: [PATCH 47/62] Fix Progress Bar in ArchlinuxImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/archlinux.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py index 342067a8a..df7c4eac2 100644 --- a/vulnerabilities/importers/archlinux.py +++ b/vulnerabilities/importers/archlinux.py @@ -39,14 +39,10 @@ def advisory_data(self) -> Iterable[AdvisoryData]: records = self.fetch() progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(records or [])) progress_bar_for_package_fetch.start() - try: - for record in records: - try: - yield from self.parse_advisory(record) - finally: - progress_bar_for_package_fetch.next() - finally: - progress_bar_for_package_fetch.finish() + for record in records: + yield from self.parse_advisory(record) + progress_bar_for_package_fetch.next() + progress_bar_for_package_fetch.finish() def parse_advisory(self, record) -> List[AdvisoryData]: advisories = [] From 728781a889274ffb7517bd88828b775af8ba4c29 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:26:24 +0530 Subject: [PATCH 48/62] Fix Progress Bar in PostgreSQLImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/postgresql.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/importers/postgresql.py b/vulnerabilities/importers/postgresql.py index 5c841b888..1120be104 100644 --- a/vulnerabilities/importers/postgresql.py +++ b/vulnerabilities/importers/postgresql.py @@ -49,14 +49,11 @@ def advisory_data(self): progress_bar_for_advisory_fetch.max = len(data_by_url) progress_bar_for_advisory_fetch.start() - try: - for url, data in data_by_url.items(): - try: - yield from to_advisories(data) - finally: - progress_bar_for_advisory_fetch.next() - finally: - progress_bar_for_advisory_fetch.finish() + for url, data in data_by_url.items(): + yield from to_advisories(data) + progress_bar_for_advisory_fetch.next() + progress_bar_for_advisory_fetch.finish() + def to_advisories(data): advisories = [] From bd760e2e1bb34031d647ef40fa7baec7c20ec5cd Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:32:22 +0530 Subject: [PATCH 49/62] Fix Progress Bar in DebianImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/debian.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py index 5965f7aa7..a7a29f6f1 100644 --- a/vulnerabilities/importers/debian.py +++ b/vulnerabilities/importers/debian.py @@ -90,16 +90,14 @@ def get_response(self): def advisory_data(self) -> Iterable[AdvisoryData]: response = self.get_response() - progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(response.items())) + progress_bar_for_package_fetch = ChargingBar( + "\tFetching Packages", max=len(response.items()) + ) progress_bar_for_package_fetch.start() - try: - for pkg_name, records in response.items(): - try: - yield from self.parse(pkg_name, records) - finally: - progress_bar_for_package_fetch.next() - finally: - progress_bar_for_package_fetch.finish() + for pkg_name, records in response.items(): + yield from self.parse(pkg_name, records) + progress_bar_for_package_fetch.next() + progress_bar_for_package_fetch.finish() def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryData]: for cve_id, record in records.items(): From 9b3b83b1e76ca24330301299ed326a02638278dd Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:38:29 +0530 Subject: [PATCH 50/62] Fix Progress Bar in OpensslImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/openssl.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/vulnerabilities/importers/openssl.py b/vulnerabilities/importers/openssl.py index d84c3ec58..ac7ee685a 100644 --- a/vulnerabilities/importers/openssl.py +++ b/vulnerabilities/importers/openssl.py @@ -52,18 +52,13 @@ def parse_vulnerabilities(xml_response) -> Iterable[AdvisoryData]: root = DET.fromstring(xml_response) progress_bar_for_vulnerability_fetch = ChargingBar("\tFetching Vulnerabilities", max=len(root)) progress_bar_for_vulnerability_fetch.start() - try: - for xml_issue in root: - try: - if xml_issue.tag == "issue": - advisory = to_advisory_data(xml_issue) - if advisory: - yield advisory - finally: - progress_bar_for_vulnerability_fetch.next() - finally: - progress_bar_for_vulnerability_fetch.finish() - + for xml_issue in root: + if xml_issue.tag == "issue": + advisory = to_advisory_data(xml_issue) + if advisory: + yield advisory + progress_bar_for_vulnerability_fetch.next() + progress_bar_for_vulnerability_fetch.finish() def to_advisory_data(xml_issue) -> AdvisoryData: From 2c1e6c8011458a35c139f4d3cb898f740f15c307 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:47:27 +0530 Subject: [PATCH 51/62] Fix Progress Bar in AlpineImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/alpine_linux.py | 38 +++++++++++------------ 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 72defa7b7..3f8ce8c1a 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -42,26 +42,24 @@ def advisory_data(self) -> Iterable[AdvisoryData]: page_response_content = fetch_response(BASE_URL).content advisory_directory_links = fetch_advisory_directory_links(page_response_content) advisory_links = [] - progress_for_package_fetch = ChargingBar("\tFetching Packages", max=len(advisory_directory_links)) - - try: - progress_for_package_fetch.start() - for advisory_directory_link in advisory_directory_links: - advisory_directory_page = fetch_response(advisory_directory_link).content - advisory_links.extend( - fetch_advisory_links(advisory_directory_page, advisory_directory_link) - ) - for link in advisory_links: - try: - record = fetch_response(link).json() - if not record["packages"]: - LOGGER.error(f'"packages" not found in {link!r}') - continue - yield from process_record(record=record, url=link) - finally: - progress_for_package_fetch.next() - finally: - progress_for_package_fetch.finish() + progress_for_package_fetch = ChargingBar( + "\tFetching Packages", max=len(advisory_directory_links) + ) + progress_for_package_fetch.start() + for advisory_directory_link in advisory_directory_links: + advisory_directory_page = fetch_response(advisory_directory_link).content + advisory_links.extend( + fetch_advisory_links(advisory_directory_page, advisory_directory_link) + ) + for link in advisory_links: + record = fetch_response(link).json() + if not record["packages"]: + LOGGER.error(f'"packages" not found in {link!r}') + continue + yield from process_record(record=record, url=link) + progress_for_package_fetch.next() + progress_for_package_fetch.finish() + def fetch_advisory_directory_links(page_response_content: str) -> List[str]: """ From 0cb4de58bd6ac654af29ffebbf5c035a540f25c8 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:50:53 +0530 Subject: [PATCH 52/62] Fix Progress Bar in PyPIImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/pysec.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/vulnerabilities/importers/pysec.py b/vulnerabilities/importers/pysec.py index 52e03283c..7dfacb5c7 100644 --- a/vulnerabilities/importers/pysec.py +++ b/vulnerabilities/importers/pysec.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) progress_bar_for_package_fetch = ChargingBar("\tFetching Packages") + class PyPIImporter(Importer): license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" spdx_license_expression = "CC-BY-4.0" @@ -39,19 +40,15 @@ def advisory_data(self) -> Iterable[AdvisoryData]: def process_zipfile_response(zip_file: ZipFile, url: str) -> Iterable[AdvisoryData]: - try: - progress_bar_for_package_fetch.start() - for file_name in zip_file.namelist(): - try: - if not file_name.startswith("PYSEC-"): - logger.error(f"Unsupported PyPI advisory data file: {file_name}") - continue - with zip_file.open(file_name) as f: - vul_info = json.load(f) - yield parse_advisory_data( - raw_data=vul_info, supported_ecosystem="pypi", advisory_url=url - ) - finally: - progress_bar_for_package_fetch.next() - finally: - progress_bar_for_package_fetch.finish() \ No newline at end of file + progress_bar_for_package_fetch.start() + for file_name in zip_file.namelist(): + if not file_name.startswith("PYSEC-"): + logger.error(f"Unsupported PyPI advisory data file: {file_name}") + continue + with zip_file.open(file_name) as f: + vul_info = json.load(f) + yield parse_advisory_data( + raw_data=vul_info, supported_ecosystem="pypi", advisory_url=url + ) + progress_bar_for_package_fetch.next() + progress_bar_for_package_fetch.finish() From 063627f0c5e4c7d7cc17cfb0bc45b3a9734a1175 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:53:03 +0530 Subject: [PATCH 53/62] Fix Progress Bar in NginxImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/nginx.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/importers/nginx.py b/vulnerabilities/importers/nginx.py index 0645e47c3..fe3daab09 100644 --- a/vulnerabilities/importers/nginx.py +++ b/vulnerabilities/importers/nginx.py @@ -54,16 +54,13 @@ def advisory_data_from_text(text): soup = BeautifulSoup(text, features="lxml") vuln_list = soup.select("li p") progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(vuln_list)) - try: - progress_bar_for_package_fetch.start() - for vuln_info in vuln_list: - try: - ngnix_adv = parse_advisory_data_from_paragraph(vuln_info) - yield to_advisory_data(ngnix_adv) - finally: - progress_bar_for_package_fetch.next() - finally: - progress_bar_for_package_fetch.finish() + progress_bar_for_package_fetch.start() + for vuln_info in vuln_list: + nginx_adv = parse_advisory_data_from_paragraph(vuln_info) + yield to_advisory_data(nginx_adv) + progress_bar_for_package_fetch.next() + progress_bar_for_package_fetch.finish() + class NginxAdvisory(NamedTuple): aliases: list From 8c821ac020cad9868d7bf87312a10b84d7f56649 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 06:58:18 +0530 Subject: [PATCH 54/62] Fix Progress Bar in PyPaImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/pypa.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/vulnerabilities/importers/pypa.py b/vulnerabilities/importers/pypa.py index 19173dddb..b5e1c82c7 100644 --- a/vulnerabilities/importers/pypa.py +++ b/vulnerabilities/importers/pypa.py @@ -35,16 +35,13 @@ def advisory_data(self) -> Iterable[AdvisoryData]: path = Path(vcs_response.dest_dir) progress_bar_for_package_fetch.max = len(dict(fork_and_get_files(base_path=path))) progress_bar_for_package_fetch.start() - for advisory_url, raw_data in fork_and_get_files(base_path=path): - try: - yield parse_advisory_data( - raw_data=raw_data, - supported_ecosystem="pypi", - advisory_url=advisory_url, - ) - finally: - progress_bar_for_package_fetch.next() + yield parse_advisory_data( + raw_data=raw_data, + supported_ecosystem="pypi", + advisory_url=advisory_url, + ) + progress_bar_for_package_fetch.next() finally: progress_bar_for_package_fetch.finish() if self.vcs_response: From 9bd44bff059bea4aadf6be6bfb8c364ba44b4d7d Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 07:02:10 +0530 Subject: [PATCH 55/62] Fix Progress Bar in NpmImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/npm.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index b8df1478e..3fc8c09d0 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -46,12 +46,9 @@ def advisory_data(self) -> Iterable[AdvisoryData]: paths_for_files_fetched = list(npm_vulns.glob("*.json")) progress_bar_for_fetched_files.max = len(paths_for_files_fetched) progress_bar_for_fetched_files.start() - for file in paths_for_files_fetched: - try: - yield from self.to_advisory_data(file) - finally: - progress_bar_for_fetched_files.next() + yield from self.to_advisory_data(file) + progress_bar_for_fetched_files.next() finally: progress_bar_for_fetched_files.finish() if self.vcs_response: From 0f611ed437e77079f7cb2e331f16e9ef60109981 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 07:10:14 +0530 Subject: [PATCH 56/62] Fix Progress Bar in GitLabAPIImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/gitlab.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/importers/gitlab.py index 2228fd97a..3e7acec35 100644 --- a/vulnerabilities/importers/gitlab.py +++ b/vulnerabilities/importers/gitlab.py @@ -64,20 +64,18 @@ def advisory_data(self, _keep_clone=False) -> Iterable[AdvisoryData]: progress_bar_for_package_fetch.max = len(file_paths_for_fetched_files) progress_bar_for_package_fetch.start() for file_path in file_paths_for_fetched_files: - try: - gitlab_type, package_slug, vuln_id = parse_advisory_path( - base_path=base_path, - file_path=file_path, - ) + gitlab_type, package_slug, vuln_id = parse_advisory_path( + base_path=base_path, + file_path=file_path, + ) - if gitlab_type in PURL_TYPE_BY_GITLAB_SCHEME: - yield parse_gitlab_advisory(file=file_path, base_path=base_path) + if gitlab_type in PURL_TYPE_BY_GITLAB_SCHEME: + yield parse_gitlab_advisory(file=file_path, base_path=base_path) - else: - logger.error(f"Unknow package type {gitlab_type!r} in {file_path!r}") - continue - finally: - progress_bar_for_package_fetch.next() + else: + logger.error(f"Unknow package type {gitlab_type!r} in {file_path!r}") + continue + progress_bar_for_package_fetch.next() finally: progress_bar_for_package_fetch.finish() if self.vcs_response and not _keep_clone: From 87f1b2720f03b1d3253b07cc11c9b147fbde69db Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 07:11:10 +0530 Subject: [PATCH 57/62] Fix Progress Bar in GitHubAPIImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/github.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py index 2f845533f..750463f8d 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/importers/github.py @@ -100,11 +100,9 @@ class GitHubAPIImporter(Importer): def advisory_data(self) -> Iterable[AdvisoryData]: progress_bar_for_package_fetch.start() - try: - for ecosystem, package_type in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items(): - yield from send_graphql_query(ecosystem, package_type) - finally: - progress_bar_for_package_fetch.finish() + for ecosystem, package_type in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items(): + yield from send_graphql_query(ecosystem, package_type) + progress_bar_for_package_fetch.finish() def send_graphql_query(ecosystem: str, package_type: str) -> Iterable[AdvisoryData]: From 411f10ea7f32efe281543a1d18bdca2b01e98d82 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 07:14:21 +0530 Subject: [PATCH 58/62] Fix Progress Bar in NVDImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/nvd.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/vulnerabilities/importers/nvd.py b/vulnerabilities/importers/nvd.py index cb0886592..b51b443ff 100644 --- a/vulnerabilities/importers/nvd.py +++ b/vulnerabilities/importers/nvd.py @@ -10,11 +10,11 @@ import gzip import json from datetime import date -from progress.bar import ChargingBar import attr import requests from dateutil import parser as dateparser +from progress.bar import ChargingBar from vulnerabilities import severity_systems from vulnerabilities.importer import AdvisoryData @@ -80,18 +80,16 @@ def fetch_cve_data_1_1(starting_year=2002): year since ``starting_year`` defaulting to 2002. """ current_year = date.today().year - progress_bar_for_records_fetched = ChargingBar("\tRecords fetched", max=(current_year-starting_year)+1) + progress_bar_for_records_fetched = ChargingBar( + "\tRecords fetched", max=(current_year - starting_year) + 1 + ) progress_bar_for_records_fetched.start() # NVD json feeds start from 2002. - try: - for year in range(starting_year, current_year + 1): - try: - download_url = f"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{year}.json.gz" - yield year, fetch(url=download_url) - finally: - progress_bar_for_records_fetched.next() - finally: - progress_bar_for_records_fetched.finish() + for year in range(starting_year, current_year + 1): + download_url = f"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{year}.json.gz" + yield year, fetch(url=download_url) + progress_bar_for_records_fetched.next() + progress_bar_for_records_fetched.finish() def to_advisories(cve_data): From 4a867e89bc4ab3ce553fc8fab4ed5cde209940b2 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 07:15:29 +0530 Subject: [PATCH 59/62] Fix Readability in FireyeImporter Signed-off-by: Harsh Mishra --- vulnerabilities/importers/fireeye.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/importers/fireeye.py b/vulnerabilities/importers/fireeye.py index 9af5e7751..8675cbc32 100644 --- a/vulnerabilities/importers/fireeye.py +++ b/vulnerabilities/importers/fireeye.py @@ -40,9 +40,12 @@ def advisory_data(self) -> Iterable[AdvisoryData]: try: self.vcs_response = self.clone(repo_url=self.repo_url) base_path = Path(self.vcs_response.dest_dir) - files = list(filter( - lambda p: p.suffix in [".md", ".MD"], Path(self.vcs_response.dest_dir).glob("**/*") - )) + files = list( + filter( + lambda p: p.suffix in [".md", ".MD"], + Path(self.vcs_response.dest_dir).glob("**/*"), + ) + ) progress_bar_for_advisory_fetch = ChargingBar("\tFetching Advisories", max=len(files)) progress_bar_for_advisory_fetch.start() From e62be24b0760d51f2c48789b541951a4c62dfd1e Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 3 Jan 2024 07:18:38 +0530 Subject: [PATCH 60/62] Fix Readability in import.py Signed-off-by: Harsh Mishra --- vulnerabilities/management/commands/import.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/management/commands/import.py b/vulnerabilities/management/commands/import.py index 9bbb2394f..1d44ab162 100644 --- a/vulnerabilities/management/commands/import.py +++ b/vulnerabilities/management/commands/import.py @@ -11,7 +11,8 @@ import progress from django.core.management.base import BaseCommand from django.core.management.base import CommandError -from progress.bar import IncrementalBar +from progress.bar import IncrementalBar + from vulnerabilities.import_runner import ImportRunner from vulnerabilities.importers import IMPORTERS_REGISTRY From 7f5eb721e78be0f41ad57ce5db85a007e5424315 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Thu, 4 Jan 2024 19:54:35 +0530 Subject: [PATCH 61/62] Add missing dependency to setup.cfg Signed-off-by: Harsh Mishra --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 7fc65fa21..1e50aa698 100644 --- a/setup.cfg +++ b/setup.cfg @@ -73,6 +73,7 @@ install_requires = packageurl-python>=0.10.5rc1 univers>=30.10.0 license-expression>=21.6.14 + progress>=1.6 # file and data formats binaryornot>=0.4.4 From d9ea936be1cd06b56626e7f2e7aaf72671569e54 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Wed, 24 Jan 2024 19:19:15 +0530 Subject: [PATCH 62/62] Remove Print Statements from Fireeye Importer Signed-off-by: Harsh Mishra --- vulnerabilities/importers/fireeye.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vulnerabilities/importers/fireeye.py b/vulnerabilities/importers/fireeye.py index 8675cbc32..10c2f6b3d 100644 --- a/vulnerabilities/importers/fireeye.py +++ b/vulnerabilities/importers/fireeye.py @@ -76,7 +76,6 @@ def parse_advisory_data(raw_data, file, base_path) -> AdvisoryData: ) raw_data = raw_data.replace("\n\n", "\n") md_list = raw_data.split("\n") - print(md_list) md_dict = md_list_to_dict(md_list) database_id = md_list[0][1::]