boringtools · c0d3G33k · Jan 10, 2023 · Jan 10, 2023
diff --git a/.gitignore b/.gitignore
@@ -127,3 +127,10 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# IDE files
+.idea/
+.vscode/
+
+# Tool output
+/data
diff --git a/README.md b/README.md
@@ -1,2 +1,24 @@
-# git-alerts
-A Public Git repository &amp; misconfiguration detection tool
+# GitAlert
+
+GitAlert tool detects and alerts public repositories belonging to an organization and organization users that may leak any secrets along with various misconfigurations
+
+## Setup
+
+Setup GitHub personal access token as environment variable
+
+```commandline
+export GITHUB_PAT=YOUR_GITHUB_PAT
+```
+## Dependencies
+
+```commandline
+pip3 install -r requirements.txt
+```
+## Usage
+
+> Find all public GitHub repositories belonging to an organization and organization users
+
+```commandline
+python3 main.py -o your-organization-name
+```
+> For future work & support, please check the issues created
diff --git a/cmd/__init__.py b/cmd/__init__.py
diff --git a/cmd/cli.py b/cmd/cli.py
@@ -0,0 +1,15 @@
+import argparse
+
+
+def parse_cli_args():
+    parser = argparse.ArgumentParser(
+        description="GitAlert Tool"
+    )
+
+    parser.add_argument("-o", "--org", required=True)
+    parser.add_argument("-d", "--output-directory", required=False, default="/tmp")
+    args = parser.parse_args()
+
+    organization = args.org
+    output_directory = args.output_directory
+    return organization, output_directory
diff --git a/common/__init__.py b/common/__init__.py
diff --git a/common/save_to_csv.py b/common/save_to_csv.py
@@ -0,0 +1,14 @@
+import pandas
+from config.conf import *
+from logger.log import *
+
+
+def save_to_csv(filename):
+    try:
+        file_path = f"{OUTPUT_DIRECTORY}/{filename}.json"
+        get_file = open(file_path, encoding="utf-8")
+        read_file = pandas.read_json(get_file)
+        read_file.to_csv(f"{OUTPUT_DIRECTORY}/{filename}.csv", encoding="utf-8")
+        log_info(f"Saved data successfully into : {OUTPUT_DIRECTORY}/{filename}.csv")
+    except Exception as error:
+        log_error(f"error in converting JSON to CSV : {error}")
diff --git a/common/save_to_json.py b/common/save_to_json.py
@@ -0,0 +1,15 @@
+import json
+
+from config.conf import *
+from logger.log import *
+
+
+def save_to_json(data, filename):
+    try:
+        file_path = f"{OUTPUT_DIRECTORY}/{filename}.json"
+        save_data = open(file_path, "w")
+        json.dump(data, save_data)
+        save_data.close()
+        log_info(f"Saved data successfully into : {file_path}")
+    except json.decoder.JSONDecodeError as error:
+        log_error(f"Error in saving JSON data : {error}")
diff --git a/config/__init__.py b/config/__init__.py
diff --git a/config/conf.py b/config/conf.py
@@ -0,0 +1,26 @@
+from dotenv import load_dotenv
+from os import getenv
+from cmd.cli import *
+from logger.log import *
+
+
+def validate_env_variable(variable_name):
+    load_dotenv()
+    get_variable = getenv(variable_name)
+
+    if get_variable is None:
+        log_error(f"{variable_name} is not configured in the environment variable")
+        exit()
+    else:
+        log_info(f"{variable_name} is successfully configure in the environment variable")
+        return get_variable
+
+
+ORGANIZATION, OUTPUT_DIRECTORY = parse_cli_args()
+
+ORG_USERS_FILE_NAME = f"{ORGANIZATION}_users"
+ORG_REPO_FILE_NAME = f"{ORGANIZATION}_public_repositories"
+ORG_USERS_REPO_FILE_NAME = f"{ORGANIZATION}_users_public_repositories"
+
+
+
diff --git a/docs/contributing.md b/docs/contributing.md
diff --git a/docs/developer.md b/docs/developer.md
diff --git a/docs/github.md b/docs/github.md
diff --git a/github/__init__.py b/github/__init__.py
diff --git a/github/get_org_repositories.py b/github/get_org_repositories.py
@@ -0,0 +1,40 @@
+from github.get_response import *
+
+
+def get_org_repositories():
+    log_info(f"fetching {ORGANIZATION} public repositories")
+
+    try:
+        headers, response, pat_limit, link_attr = get_github_response(f"{github_url_org}/repos")
+        log_info(f"Remaining request limit: {pat_limit}")
+
+        if link_attr is None:
+            for repos in response:
+                visibility = repos["visibility"]
+                if visibility == "public":
+                    organization_directory["repository"].append(repos["html_url"])
+                    organization_directory["fork"].append(repos["fork"])
+                    organization_directory["created"].append(repos["created_at"])
+                    organization_directory["updated"].append(repos["updated_at"])
+                    organization_directory["pushed"].append(repos["pushed_at"])
+        else:
+            page_length = int(link_attr.split(",")[1].split("page=")[1].split("&")[0])
+
+            for pages in range(1, page_length + 1):
+                github_request_params["page"] = pages
+                headers, response, pat_limit, link_attr = get_github_response(f"{github_url_org}/repos")
+
+                for repos in response:
+                    visibility = repos["visibility"]
+                    if visibility == "public":
+                        organization_directory["repository"].append(repos["html_url"])
+                        organization_directory["fork"].append(repos["fork"])
+                        organization_directory["created"].append(repos["created_at"])
+                        organization_directory["updated"].append(repos["updated_at"])
+                        organization_directory["pushed"].append(repos["pushed_at"])
+
+        return organization_directory
+
+    except requests.exceptions.RequestException as error:
+        log_error(f"Error in fetching {ORGANIZATION} repositories : {error}")
+        exit()
diff --git a/github/get_org_users.py b/github/get_org_users.py
@@ -0,0 +1,30 @@
+from github.get_response import *
+
+
+def get_org_users():
+    log_info(f"Fetching {ORGANIZATION} users ")
+
+    try:
+        headers, response, pat_limit, link_attr  = get_github_response(f"{github_url_org}/members")
+        log_info(f"Remaining request limit: {pat_limit}")
+
+        if link_attr is None:
+
+            for users in response:
+                username = users["login"]
+                users_directory["usernames"].append(username)
+        else:
+            page_length = int(link_attr.split(",")[1].split("page=")[1].split("&")[0])
+
+            for pages in range(1, page_length + 1):
+                github_request_params["page"] = pages
+                headers, response, pat_limit, link_attr = get_github_response(f"{github_url_org}/members")
+
+                for users in response:
+                    username = users["login"]
+                    users_directory["usernames"].append(username)
+        return users_directory
+
+    except requests.exceptions.RequestException as error:
+        log_error(f"Error in fetching {ORGANIZATION} users : {error}")
+        exit()
diff --git a/github/get_org_users_repositories.py b/github/get_org_users_repositories.py
@@ -0,0 +1,40 @@
+import json
+
+from github.get_response import *
+
+
+def get_org_users_repositories():
+    log_info(f"Fetching {ORGANIZATION} users public repositories")
+
+    try:
+        org_users = open(f"{OUTPUT_DIRECTORY}/{ORG_USERS_FILE_NAME}.json", "r")
+        load_org_users = json.load(org_users)
+        users = load_org_users["usernames"]
+
+        for user in users:
+            headers, response, pat_limit, link_attr = get_github_response(f"{github_url_user}/{user}/repos")
+            if not link_attr:
+                for repo in response:
+                    organization_users_directory["repository"].append(repo["html_url"])
+                    organization_users_directory["fork"].append(repo["fork"])
+                    organization_users_directory["created"].append(repo["created_at"])
+                    organization_users_directory["updated"].append(repo["updated_at"])
+                    organization_users_directory["pushed"].append(repo["pushed_at"])
+            else:
+                page_length = int(link_attr.split(",")[1].split("page=")[1].split("&")[0])
+                for pages in range(1, page_length + 1):
+                    github_request_params["page"] = pages
+                    headers_2, response_2, pat_limit_2, link_attr_2 = get_github_response(f"{github_url_user}/{user}/repos")
+                    for repos in response_2:
+                        organization_users_directory["repository"].append(repos["html_url"])
+                        organization_users_directory["fork"].append(repos["fork"])
+                        organization_users_directory["created"].append(repos["created_at"])
+                        organization_users_directory["updated"].append(repos["updated_at"])
+                        organization_users_directory["pushed"].append(repos["pushed_at"])
+
+        log_info(f"Remaining request limit: {pat_limit}")
+        return organization_users_directory
+
+    except json.decoder.JSONDecodeError as error:
+        log_error(f"Error in reading JSON data : {error}")
+        exit()
diff --git a/github/get_response.py b/github/get_response.py
@@ -0,0 +1,21 @@
+import requests
+from github.github_config import *
+
+
+def get_github_response(url):
+    try:
+        get_data = requests.get(url, params=github_request_params, headers=GITHUB_REQUEST_HEADERS)
+        get_data_headers = get_data.headers
+        get_data_response = get_data.json()
+
+        pat_request_limit = get_data_headers["X-RateLimit-Remaining"]
+        check_link_attr = get_data_headers.get("Link")
+
+        if int(pat_request_limit) < 20:
+            log_error("GitHub PAT request limit reached")
+            exit()
+
+        return get_data_headers, get_data_response, pat_request_limit, check_link_attr
+
+    except requests.exceptions.RequestException as error:
+        log_error(f"Failed to get response to the URL : {url}, ERROR: {error}  ")
diff --git a/github/github_config.py b/github/github_config.py
@@ -0,0 +1,13 @@
+from config.conf import *
+
+GITHUB_PAT = validate_env_variable("GITHUB_PAT")
+GITHUB_REQUEST_HEADERS = {"Authorization": f"token {GITHUB_PAT}"}
+
+users_directory = {"usernames": []}
+organization_directory = {"repository": [], "fork": [], "created": [], "updated": [], "pushed": []}
+organization_users_directory = {"repository": [], "fork": [], "created": [], "updated": [], "pushed": []}
+github_request_params = {"page": "1", "per_page": "100"}
+
+github_url_org = f"https://api.github.com/orgs/{ORGANIZATION}"
+github_url_user = f"https://api.github.com/users"
+
diff --git a/logger/__init__.py b/logger/__init__.py
diff --git a/logger/log.py b/logger/log.py
@@ -0,0 +1,13 @@
+def log_info(value):
+    print(f"[+] {value}")
+    return
+
+
+def log_warn(value):
+    print(f"[WARN] {value}")
+    return
+
+
+def log_error(value):
+    print(f"[ERROR] {value}")
+    return
diff --git a/main.py b/main.py
@@ -0,0 +1,23 @@
+from github.get_org_users import *
+from github.get_org_repositories import *
+from github.get_org_users_repositories import *
+from common.save_to_csv import *
+from common.save_to_json import *
+
+
+def main():
+    org_users = get_org_users()
+    save_to_json(org_users, ORG_USERS_FILE_NAME)
+    save_to_csv(ORG_USERS_FILE_NAME)
+
+    org_repo = get_org_repositories()
+    save_to_json(org_repo, ORG_REPO_FILE_NAME)
+    save_to_csv(ORG_REPO_FILE_NAME)
+
+    org_user_repo = get_org_users_repositories()
+    save_to_json(org_user_repo, ORG_USERS_REPO_FILE_NAME)
+    save_to_csv(ORG_USERS_REPO_FILE_NAME)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/notification/__init__.py b/notification/__init__.py
diff --git a/notification/email.py b/notification/email.py
diff --git a/notification/message.py b/notification/message.py
diff --git a/notification/slack.py b/notification/slack.py
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,4 @@
+argparse~=1.4.0
+python-dotenv~=0.21.0
+requests~=2.28.1
+pandas~=1.5.2