aboutcode-org · harsh098 · Dec 25, 2023 · Dec 26, 2023 · Dec 27, 2023 · Dec 27, 2023
diff --git a/requirements.txt b/requirements.txt
@@ -120,3 +120,4 @@ drf-spectacular==0.24.2
 coreapi==2.3.3
 coreschema==0.0.4
 itypes==1.2.0
+progress==1.6
diff --git a/setup.cfg b/setup.cfg
@@ -73,6 +73,7 @@ install_requires =
     packageurl-python>=0.10.5rc1
     univers>=30.11.0
     license-expression>=21.6.14
+    progress>=1.6
 
     # file and data formats
     binaryornot>=0.4.4

diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py
@@ -17,6 +17,7 @@
 
 from bs4 import BeautifulSoup
 from packageurl import PackageURL
+from progress.bar import ChargingBar
 from univers.versions import AlpineLinuxVersion
 
 from vulnerabilities.importer import AdvisoryData
@@ -41,6 +42,10 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
         page_response_content = fetch_response(BASE_URL).content
         advisory_directory_links = fetch_advisory_directory_links(page_response_content)
         advisory_links = []
+        progress_for_package_fetch = ChargingBar(
+            "\tFetching Packages", max=len(advisory_directory_links)
+        )
+        progress_for_package_fetch.start()
         for advisory_directory_link in advisory_directory_links:
             advisory_directory_page = fetch_response(advisory_directory_link).content
             advisory_links.extend(
@@ -52,6 +57,8 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
                 LOGGER.error(f'"packages" not found in {link!r}')
                 continue
             yield from process_record(record=record, url=link)
+            progress_for_package_fetch.next()
+        progress_for_package_fetch.finish()
 
 
 def fetch_advisory_directory_links(page_response_content: str) -> List[str]:

diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py
@@ -13,6 +13,7 @@
 import requests
 from bs4 import BeautifulSoup
 from packageurl import PackageURL
+from progress.bar import ChargingBar
 from univers.version_constraint import VersionConstraint
 from univers.version_range import ApacheVersionRange
 from univers.versions import SemverVersion
@@ -37,9 +38,13 @@ class ApacheHTTPDImporter(Importer):
 
     def advisory_data(self):
         links = fetch_links(self.base_url)
+        progress_bar_for_fetch_links = ChargingBar("\tFetching Vulnerabilitites", max=len(links))
+        progress_bar_for_fetch_links.start()
         for link in links:
             data = requests.get(link).json()
             yield self.to_advisory(data)
+            progress_bar_for_fetch_links.next()
+        progress_bar_for_fetch_links.finish()
 
     def to_advisory(self, data):
         alias = get_item(data, "CVE_data_meta", "ID")

diff --git a/vulnerabilities/importers/apache_kafka.py b/vulnerabilities/importers/apache_kafka.py
@@ -15,6 +15,7 @@
 from bs4 import BeautifulSoup
 from dateutil.parser import parse
 from packageurl import PackageURL
+from progress.bar import ChargingBar
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
@@ -113,6 +114,8 @@ def to_advisory(self, advisory_page):
 
         advisory_page = BeautifulSoup(advisory_page, features="lxml")
         cve_section_beginnings = advisory_page.find_all("h2")
+        progress_bar_for_cve_fetch = ChargingBar("\tFetching CVEs", max=len(cve_section_beginnings))
+        progress_bar_for_cve_fetch.start()
         for cve_section_beginning in cve_section_beginnings:
             # This sometimes includes text that follows the CVE on the same line -- sometimes there is a carriage return, sometimes there is not
             # cve_id = cve_section_beginning.text.split("\n")[0]
@@ -195,5 +198,8 @@ def to_advisory(self, advisory_page):
                         url=f"{self.ASF_PAGE_URL}#{cve_id}",
                     )
                 )
+            progress_bar_for_cve_fetch.next()
+
+        progress_bar_for_cve_fetch.finish()
 
         return advisories
diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py
@@ -15,6 +15,7 @@
 import requests
 from bs4 import BeautifulSoup
 from packageurl import PackageURL
+from progress.bar import ChargingBar
 from univers.version_constraint import VersionConstraint
 from univers.version_range import ApacheVersionRange
 from univers.version_range import MavenVersionRange
@@ -124,9 +125,14 @@ def fetch_advisory_pages(self):
         """
         Yield the content of each HTML page containing version-related security data.
         """
-        links = self.fetch_advisory_links("https://tomcat.apache.org/security")
+        links = list(self.fetch_advisory_links("https://tomcat.apache.org/security"))
+        progress_bar_for_advisory_fetch = ChargingBar("\tFetching Advisories", max=len(links))
+        progress_bar_for_advisory_fetch.start()
         for page_url in links:
             yield page_url, requests.get(page_url).content
+            progress_bar_for_advisory_fetch.next()
+
+        progress_bar_for_advisory_fetch.finish()
 
     def fetch_advisory_links(self, url):
         """

diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py
@@ -12,6 +12,7 @@
 from typing import Mapping
 
 from packageurl import PackageURL
+from progress.bar import ChargingBar
 from univers.version_range import ArchLinuxVersionRange
 from univers.versions import ArchLinuxVersion
 
@@ -35,8 +36,13 @@ def fetch(self) -> Iterable[Mapping]:
         return response.json()
 
     def advisory_data(self) -> Iterable[AdvisoryData]:
-        for record in self.fetch():
+        records = self.fetch()
+        progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(records or []))
+        progress_bar_for_package_fetch.start()
+        for record in records:
             yield from self.parse_advisory(record)
+            progress_bar_for_package_fetch.next()
+        progress_bar_for_package_fetch.finish()
 
     def parse_advisory(self, record) -> List[AdvisoryData]:
         advisories = []

diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py
@@ -15,6 +15,7 @@
 
 import requests
 from packageurl import PackageURL
+from progress.bar import ChargingBar
 from univers.version_range import DebianVersionRange
 from univers.versions import DebianVersion
 
@@ -89,8 +90,14 @@ def get_response(self):
 
     def advisory_data(self) -> Iterable[AdvisoryData]:
         response = self.get_response()
+        progress_bar_for_package_fetch = ChargingBar(
+            "\tFetching Packages", max=len(response.items())
+        )
+        progress_bar_for_package_fetch.start()
         for pkg_name, records in response.items():
             yield from self.parse(pkg_name, records)
+            progress_bar_for_package_fetch.next()
+        progress_bar_for_package_fetch.finish()
 
     def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryData]:
         for cve_id, record in records.items():

diff --git a/vulnerabilities/importers/debian_oval.py b/vulnerabilities/importers/debian_oval.py
@@ -12,6 +12,7 @@
 import xml.etree.ElementTree as ET
 
 import requests
+from progress.bar import ChargingBar
 
 from vulnerabilities.importer import OvalImporter
 
@@ -65,6 +66,8 @@ def __init__(self, *args, **kwargs):
 
     def _fetch(self):
         releases = ["wheezy", "stretch", "jessie", "buster", "bullseye"]
+        progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(releases))
+        progress_bar_for_package_fetch.start()
         for release in releases:
             file_url = f"https://www.debian.org/security/oval/oval-definitions-{release}.xml.bz2"
             self.data_url = file_url
@@ -74,3 +77,5 @@ def _fetch(self):
                 {"type": "deb", "namespace": "debian", "qualifiers": {"distro": release}},
                 ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))),
             )
+            progress_bar_for_package_fetch.next()
+        progress_bar_for_package_fetch.finish()
diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py
@@ -12,6 +12,7 @@
 
 from dateutil import parser as dateparser
 from packageurl import PackageURL
+from progress.bar import ChargingBar
 from univers.version_constraint import VersionConstraint
 from univers.version_range import HexVersionRange
 
@@ -31,13 +32,19 @@ class ElixirSecurityImporter(Importer):
     importer_name = "Elixir Security Importer"
 
     def advisory_data(self) -> Set[AdvisoryData]:
+        progress_bar_for_cve_fetch: ChargingBar
         try:
             self.clone(self.repo_url)
             base_path = Path(self.vcs_response.dest_dir)
             vuln = base_path / "packages"
-            for file in vuln.glob("**/*.yml"):
+            vuln_files = list(vuln.glob("**/*.yml"))
+            progress_bar_for_cve_fetch = ChargingBar("\tFetching CVEs", max=len(vuln_files))
+            progress_bar_for_cve_fetch.start()
+            for file in vuln_files:
                 yield from self.process_file(file, base_path)
+                progress_bar_for_cve_fetch.next()
         finally:
+            progress_bar_for_cve_fetch.finish()
             if self.vcs_response:
                 self.vcs_response.delete()
 

diff --git a/vulnerabilities/importers/fireeye.py b/vulnerabilities/importers/fireeye.py
@@ -12,6 +12,8 @@
 from typing import Iterable
 from typing import List
 
+from progress.bar import ChargingBar
+
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
@@ -34,12 +36,19 @@ class FireyeImporter(Importer):
     importer_name = "FireEye Importer"
 
     def advisory_data(self) -> Iterable[AdvisoryData]:
+        progress_bar_for_advisory_fetch: ChargingBar
         try:
             self.vcs_response = self.clone(repo_url=self.repo_url)
             base_path = Path(self.vcs_response.dest_dir)
-            files = filter(
-                lambda p: p.suffix in [".md", ".MD"], Path(self.vcs_response.dest_dir).glob("**/*")
+            files = list(
+                filter(
+                    lambda p: p.suffix in [".md", ".MD"],
+                    Path(self.vcs_response.dest_dir).glob("**/*"),
+                )
             )
+            progress_bar_for_advisory_fetch = ChargingBar("\tFetching Advisories", max=len(files))
+            progress_bar_for_advisory_fetch.start()
+
             for file in files:
                 if Path(file).stem == "README":
                     continue
@@ -48,7 +57,10 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
                         yield parse_advisory_data(raw_data=f.read(), file=file, base_path=base_path)
                 except UnicodeError:
                     logger.error(f"Invalid file {file}")
+                finally:
+                    progress_bar_for_advisory_fetch.next()
         finally:
+            progress_bar_for_advisory_fetch.finish()
             if self.vcs_response:
                 self.vcs_response.delete()
 

diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py
@@ -14,6 +14,7 @@
 from typing import Iterable
 
 from packageurl import PackageURL
+from progress.bar import ChargingBar
 from univers.version_constraint import VersionConstraint
 from univers.version_range import EbuildVersionRange
 from univers.versions import GentooVersion
@@ -34,12 +35,18 @@ class GentooImporter(Importer):
     importer_name = "Gentoo Importer"
 
     def advisory_data(self) -> Iterable[AdvisoryData]:
+        progress_bar_for_package_fetch: ChargingBar
         try:
             self.clone(repo_url=self.repo_url)
             base_path = Path(self.vcs_response.dest_dir)
-            for file_path in base_path.glob("**/*.xml"):
+            base_paths = list(base_path.glob("**/*.xml"))
+            progress_bar_for_package_fetch = ChargingBar("\tFetching Packages", max=len(base_paths))
+            progress_bar_for_package_fetch.start()
+            for file_path in base_paths:
                 yield from self.process_file(file_path)
+                progress_bar_for_package_fetch.next()
         finally:
+            progress_bar_for_package_fetch.finish()
             if self.vcs_response:
                 self.vcs_response.delete()
 

diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py
@@ -14,6 +14,7 @@
 from cwe2.database import Database
 from dateutil import parser as dateparser
 from packageurl import PackageURL
+from progress.bar import ChargingBar
 from univers.version_range import RANGE_CLASS_BY_SCHEMES
 from univers.version_range import build_range_from_github_advisory_constraint
 
@@ -88,29 +89,42 @@
 }
 """
 
+progress_bar_for_package_fetch = ChargingBar(
+    "\tFetching Packages", max=len(PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items())
+)
+
 
 class GitHubAPIImporter(Importer):
     spdx_license_expression = "CC-BY-4.0"
     importer_name = "GHSA Importer"
     license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
 
     def advisory_data(self) -> Iterable[AdvisoryData]:
+        progress_bar_for_package_fetch.start()
         for ecosystem, package_type in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items():
-            end_cursor_exp = ""
-            while True:
-                graphql_query = {"query": GRAPHQL_QUERY_TEMPLATE % (ecosystem, end_cursor_exp)}
-                response = utils.fetch_github_graphql_query(graphql_query)
+            yield from send_graphql_query(ecosystem, package_type)
+        progress_bar_for_package_fetch.finish()
+
+
+def send_graphql_query(ecosystem: str, package_type: str) -> Iterable[AdvisoryData]:
+    try:
+        end_cursor_exp = ""
+        while True:
+            graphql_query = {"query": GRAPHQL_QUERY_TEMPLATE % (ecosystem, end_cursor_exp)}
+            response = utils.fetch_github_graphql_query(graphql_query)
 
-                page_info = get_item(response, "data", "securityVulnerabilities", "pageInfo")
-                end_cursor = get_item(page_info, "endCursor")
-                if end_cursor:
-                    end_cursor = f'"{end_cursor}"'
-                    end_cursor_exp = f"after: {end_cursor}"
+            page_info = get_item(response, "data", "securityVulnerabilities", "pageInfo")
+            end_cursor = get_item(page_info, "endCursor")
+            if end_cursor:
+                end_cursor = f'"{end_cursor}"'
+                end_cursor_exp = f"after: {end_cursor}"
 
-                yield from process_response(response, package_type=package_type)
+            yield from process_response(response, package_type=package_type)
 
-                if not get_item(page_info, "hasNextPage"):
-                    break
+            if not get_item(page_info, "hasNextPage"):
+                break
+    finally:
+        progress_bar_for_package_fetch.next()
 
 
 def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]:

diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/importers/gitlab.py
@@ -18,6 +18,7 @@
 import saneyaml
 from dateutil import parser as dateparser
 from packageurl import PackageURL
+from progress.bar import ChargingBar
 from univers.version_range import RANGE_CLASS_BY_SCHEMES
 from univers.version_range import VersionRange
 from univers.version_range import from_gitlab_native
@@ -55,11 +56,14 @@ class GitLabAPIImporter(Importer):
     repo_url = "git+https://gitlab.com/gitlab-org/advisories-community/"
 
     def advisory_data(self, _keep_clone=False) -> Iterable[AdvisoryData]:
+        progress_bar_for_package_fetch = ChargingBar("\tFetching Packages")
         try:
             self.clone(repo_url=self.repo_url)
             base_path = Path(self.vcs_response.dest_dir)
-
-            for file_path in base_path.glob("**/*.yml"):
+            file_paths_for_fetched_files = list(base_path.glob("**/*.yml"))
+            progress_bar_for_package_fetch.max = len(file_paths_for_fetched_files)
+            progress_bar_for_package_fetch.start()
+            for file_path in file_paths_for_fetched_files:
                 gitlab_type, package_slug, vuln_id = parse_advisory_path(
                     base_path=base_path,
                     file_path=file_path,
@@ -71,7 +75,9 @@ def advisory_data(self, _keep_clone=False) -> Iterable[AdvisoryData]:
                 else:
                     logger.error(f"Unknow package type {gitlab_type!r} in {file_path!r}")
                     continue
+                progress_bar_for_package_fetch.next()
         finally:
+            progress_bar_for_package_fetch.finish()
             if self.vcs_response and not _keep_clone:
                 self.vcs_response.delete()