From 276906e1af5a0bfcdc6ffea89769eaf343acccd9 Mon Sep 17 00:00:00 2001 From: c0d3G33k Date: Tue, 10 Jan 2023 23:33:26 +0530 Subject: [PATCH] inital release --- .gitignore | 7 +++++ README.md | 26 ++++++++++++++++-- cmd/__init__.py | 0 cmd/cli.py | 15 +++++++++++ common/__init__.py | 0 common/save_to_csv.py | 14 ++++++++++ common/save_to_json.py | 15 +++++++++++ config/__init__.py | 0 config/conf.py | 26 ++++++++++++++++++ docs/contributing.md | 0 docs/developer.md | 0 docs/github.md | 0 github/__init__.py | 0 github/get_org_repositories.py | 40 ++++++++++++++++++++++++++++ github/get_org_users.py | 30 +++++++++++++++++++++ github/get_org_users_repositories.py | 40 ++++++++++++++++++++++++++++ github/get_response.py | 21 +++++++++++++++ github/github_config.py | 13 +++++++++ logger/__init__.py | 0 logger/log.py | 13 +++++++++ main.py | 23 ++++++++++++++++ notification/__init__.py | 0 notification/email.py | 0 notification/message.py | 0 notification/slack.py | 0 requirements.txt | 4 +++ 26 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 cmd/__init__.py create mode 100644 cmd/cli.py create mode 100644 common/__init__.py create mode 100644 common/save_to_csv.py create mode 100644 common/save_to_json.py create mode 100644 config/__init__.py create mode 100644 config/conf.py create mode 100644 docs/contributing.md create mode 100644 docs/developer.md create mode 100644 docs/github.md create mode 100644 github/__init__.py create mode 100644 github/get_org_repositories.py create mode 100644 github/get_org_users.py create mode 100644 github/get_org_users_repositories.py create mode 100644 github/get_response.py create mode 100644 github/github_config.py create mode 100644 logger/__init__.py create mode 100644 logger/log.py create mode 100644 main.py create mode 100644 notification/__init__.py create mode 100644 notification/email.py create mode 100644 notification/message.py create mode 100644 notification/slack.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index b6e4761..5ba53d1 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,10 @@ dmypy.json # Pyre type checker .pyre/ + +# IDE files +.idea/ +.vscode/ + +# Tool output +/data diff --git a/README.md b/README.md index b839a70..0637baa 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,24 @@ -# git-alerts -A Public Git repository & misconfiguration detection tool +# GitAlert + +GitAlert tool detects and alerts public repositories belonging to an organization and organization users that may leak any secrets along with various misconfigurations + +## Setup + +Setup GitHub personal access token as environment variable + +```commandline +export GITHUB_PAT=YOUR_GITHUB_PAT +``` +## Dependencies + +```commandline +pip3 install -r requirements.txt +``` +## Usage + +> Find all public GitHub repositories belonging to an organization and organization users + +```commandline +python3 main.py -o your-organization-name +``` +> For future work & support, please check the issues created \ No newline at end of file diff --git a/cmd/__init__.py b/cmd/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cmd/cli.py b/cmd/cli.py new file mode 100644 index 0000000..802d847 --- /dev/null +++ b/cmd/cli.py @@ -0,0 +1,15 @@ +import argparse + + +def parse_cli_args(): + parser = argparse.ArgumentParser( + description="GitAlert Tool" + ) + + parser.add_argument("-o", "--org", required=True) + parser.add_argument("-d", "--output-directory", required=False, default="/tmp") + args = parser.parse_args() + + organization = args.org + output_directory = args.output_directory + return organization, output_directory diff --git a/common/__init__.py b/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/common/save_to_csv.py b/common/save_to_csv.py new file mode 100644 index 0000000..192668f --- /dev/null +++ b/common/save_to_csv.py @@ -0,0 +1,14 @@ +import pandas +from config.conf import * +from logger.log import * + + +def save_to_csv(filename): + try: + file_path = f"{OUTPUT_DIRECTORY}/{filename}.json" + get_file = open(file_path, encoding="utf-8") + read_file = pandas.read_json(get_file) + read_file.to_csv(f"{OUTPUT_DIRECTORY}/{filename}.csv", encoding="utf-8") + log_info(f"Saved data successfully into : {OUTPUT_DIRECTORY}/{filename}.csv") + except Exception as error: + log_error(f"error in converting JSON to CSV : {error}") diff --git a/common/save_to_json.py b/common/save_to_json.py new file mode 100644 index 0000000..a4613ae --- /dev/null +++ b/common/save_to_json.py @@ -0,0 +1,15 @@ +import json + +from config.conf import * +from logger.log import * + + +def save_to_json(data, filename): + try: + file_path = f"{OUTPUT_DIRECTORY}/{filename}.json" + save_data = open(file_path, "w") + json.dump(data, save_data) + save_data.close() + log_info(f"Saved data successfully into : {file_path}") + except json.decoder.JSONDecodeError as error: + log_error(f"Error in saving JSON data : {error}") \ No newline at end of file diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/conf.py b/config/conf.py new file mode 100644 index 0000000..d825a9b --- /dev/null +++ b/config/conf.py @@ -0,0 +1,26 @@ +from dotenv import load_dotenv +from os import getenv +from cmd.cli import * +from logger.log import * + + +def validate_env_variable(variable_name): + load_dotenv() + get_variable = getenv(variable_name) + + if get_variable is None: + log_error(f"{variable_name} is not configured in the environment variable") + exit() + else: + log_info(f"{variable_name} is successfully configure in the environment variable") + return get_variable + + +ORGANIZATION, OUTPUT_DIRECTORY = parse_cli_args() + +ORG_USERS_FILE_NAME = f"{ORGANIZATION}_users" +ORG_REPO_FILE_NAME = f"{ORGANIZATION}_public_repositories" +ORG_USERS_REPO_FILE_NAME = f"{ORGANIZATION}_users_public_repositories" + + + diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/developer.md b/docs/developer.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/github.md b/docs/github.md new file mode 100644 index 0000000..e69de29 diff --git a/github/__init__.py b/github/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/github/get_org_repositories.py b/github/get_org_repositories.py new file mode 100644 index 0000000..065da15 --- /dev/null +++ b/github/get_org_repositories.py @@ -0,0 +1,40 @@ +from github.get_response import * + + +def get_org_repositories(): + log_info(f"fetching {ORGANIZATION} public repositories") + + try: + headers, response, pat_limit, link_attr = get_github_response(f"{github_url_org}/repos") + log_info(f"Remaining request limit: {pat_limit}") + + if link_attr is None: + for repos in response: + visibility = repos["visibility"] + if visibility == "public": + organization_directory["repository"].append(repos["html_url"]) + organization_directory["fork"].append(repos["fork"]) + organization_directory["created"].append(repos["created_at"]) + organization_directory["updated"].append(repos["updated_at"]) + organization_directory["pushed"].append(repos["pushed_at"]) + else: + page_length = int(link_attr.split(",")[1].split("page=")[1].split("&")[0]) + + for pages in range(1, page_length + 1): + github_request_params["page"] = pages + headers, response, pat_limit, link_attr = get_github_response(f"{github_url_org}/repos") + + for repos in response: + visibility = repos["visibility"] + if visibility == "public": + organization_directory["repository"].append(repos["html_url"]) + organization_directory["fork"].append(repos["fork"]) + organization_directory["created"].append(repos["created_at"]) + organization_directory["updated"].append(repos["updated_at"]) + organization_directory["pushed"].append(repos["pushed_at"]) + + return organization_directory + + except requests.exceptions.RequestException as error: + log_error(f"Error in fetching {ORGANIZATION} repositories : {error}") + exit() diff --git a/github/get_org_users.py b/github/get_org_users.py new file mode 100644 index 0000000..ca5b9bf --- /dev/null +++ b/github/get_org_users.py @@ -0,0 +1,30 @@ +from github.get_response import * + + +def get_org_users(): + log_info(f"Fetching {ORGANIZATION} users ") + + try: + headers, response, pat_limit, link_attr = get_github_response(f"{github_url_org}/members") + log_info(f"Remaining request limit: {pat_limit}") + + if link_attr is None: + + for users in response: + username = users["login"] + users_directory["usernames"].append(username) + else: + page_length = int(link_attr.split(",")[1].split("page=")[1].split("&")[0]) + + for pages in range(1, page_length + 1): + github_request_params["page"] = pages + headers, response, pat_limit, link_attr = get_github_response(f"{github_url_org}/members") + + for users in response: + username = users["login"] + users_directory["usernames"].append(username) + return users_directory + + except requests.exceptions.RequestException as error: + log_error(f"Error in fetching {ORGANIZATION} users : {error}") + exit() diff --git a/github/get_org_users_repositories.py b/github/get_org_users_repositories.py new file mode 100644 index 0000000..4869baf --- /dev/null +++ b/github/get_org_users_repositories.py @@ -0,0 +1,40 @@ +import json + +from github.get_response import * + + +def get_org_users_repositories(): + log_info(f"Fetching {ORGANIZATION} users public repositories") + + try: + org_users = open(f"{OUTPUT_DIRECTORY}/{ORG_USERS_FILE_NAME}.json", "r") + load_org_users = json.load(org_users) + users = load_org_users["usernames"] + + for user in users: + headers, response, pat_limit, link_attr = get_github_response(f"{github_url_user}/{user}/repos") + if not link_attr: + for repo in response: + organization_users_directory["repository"].append(repo["html_url"]) + organization_users_directory["fork"].append(repo["fork"]) + organization_users_directory["created"].append(repo["created_at"]) + organization_users_directory["updated"].append(repo["updated_at"]) + organization_users_directory["pushed"].append(repo["pushed_at"]) + else: + page_length = int(link_attr.split(",")[1].split("page=")[1].split("&")[0]) + for pages in range(1, page_length + 1): + github_request_params["page"] = pages + headers_2, response_2, pat_limit_2, link_attr_2 = get_github_response(f"{github_url_user}/{user}/repos") + for repos in response_2: + organization_users_directory["repository"].append(repos["html_url"]) + organization_users_directory["fork"].append(repos["fork"]) + organization_users_directory["created"].append(repos["created_at"]) + organization_users_directory["updated"].append(repos["updated_at"]) + organization_users_directory["pushed"].append(repos["pushed_at"]) + + log_info(f"Remaining request limit: {pat_limit}") + return organization_users_directory + + except json.decoder.JSONDecodeError as error: + log_error(f"Error in reading JSON data : {error}") + exit() diff --git a/github/get_response.py b/github/get_response.py new file mode 100644 index 0000000..d15b050 --- /dev/null +++ b/github/get_response.py @@ -0,0 +1,21 @@ +import requests +from github.github_config import * + + +def get_github_response(url): + try: + get_data = requests.get(url, params=github_request_params, headers=GITHUB_REQUEST_HEADERS) + get_data_headers = get_data.headers + get_data_response = get_data.json() + + pat_request_limit = get_data_headers["X-RateLimit-Remaining"] + check_link_attr = get_data_headers.get("Link") + + if int(pat_request_limit) < 20: + log_error("GitHub PAT request limit reached") + exit() + + return get_data_headers, get_data_response, pat_request_limit, check_link_attr + + except requests.exceptions.RequestException as error: + log_error(f"Failed to get response to the URL : {url}, ERROR: {error} ") diff --git a/github/github_config.py b/github/github_config.py new file mode 100644 index 0000000..336b638 --- /dev/null +++ b/github/github_config.py @@ -0,0 +1,13 @@ +from config.conf import * + +GITHUB_PAT = validate_env_variable("GITHUB_PAT") +GITHUB_REQUEST_HEADERS = {"Authorization": f"token {GITHUB_PAT}"} + +users_directory = {"usernames": []} +organization_directory = {"repository": [], "fork": [], "created": [], "updated": [], "pushed": []} +organization_users_directory = {"repository": [], "fork": [], "created": [], "updated": [], "pushed": []} +github_request_params = {"page": "1", "per_page": "100"} + +github_url_org = f"https://api.github.com/orgs/{ORGANIZATION}" +github_url_user = f"https://api.github.com/users" + diff --git a/logger/__init__.py b/logger/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/logger/log.py b/logger/log.py new file mode 100644 index 0000000..5fec6f8 --- /dev/null +++ b/logger/log.py @@ -0,0 +1,13 @@ +def log_info(value): + print(f"[+] {value}") + return + + +def log_warn(value): + print(f"[WARN] {value}") + return + + +def log_error(value): + print(f"[ERROR] {value}") + return diff --git a/main.py b/main.py new file mode 100644 index 0000000..31ccd2e --- /dev/null +++ b/main.py @@ -0,0 +1,23 @@ +from github.get_org_users import * +from github.get_org_repositories import * +from github.get_org_users_repositories import * +from common.save_to_csv import * +from common.save_to_json import * + + +def main(): + org_users = get_org_users() + save_to_json(org_users, ORG_USERS_FILE_NAME) + save_to_csv(ORG_USERS_FILE_NAME) + + org_repo = get_org_repositories() + save_to_json(org_repo, ORG_REPO_FILE_NAME) + save_to_csv(ORG_REPO_FILE_NAME) + + org_user_repo = get_org_users_repositories() + save_to_json(org_user_repo, ORG_USERS_REPO_FILE_NAME) + save_to_csv(ORG_USERS_REPO_FILE_NAME) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/notification/__init__.py b/notification/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/notification/email.py b/notification/email.py new file mode 100644 index 0000000..e69de29 diff --git a/notification/message.py b/notification/message.py new file mode 100644 index 0000000..e69de29 diff --git a/notification/slack.py b/notification/slack.py new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5d6becb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +argparse~=1.4.0 +python-dotenv~=0.21.0 +requests~=2.28.1 +pandas~=1.5.2 \ No newline at end of file