From 5ee80c83bdca2faa34a0df1c93f6c95ad3239758 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 20:12:05 +0000 Subject: [PATCH 01/14] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/publish_lists.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_lists.yml b/.github/workflows/publish_lists.yml index a8c3d8f..a5f1b0b 100644 --- a/.github/workflows/publish_lists.yml +++ b/.github/workflows/publish_lists.yml @@ -1,4 +1,4 @@ -ο»Ώname: Publish Lists +name: Publish Lists on: schedule: @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 From c1b7a898df8d7b4f54e0e93def87fb6e5cc1eab6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 20:04:55 +0000 Subject: [PATCH 02/14] Bump stefanzweifel/git-auto-commit-action from 4 to 5 Bumps [stefanzweifel/git-auto-commit-action](https://github.com/stefanzweifel/git-auto-commit-action) from 4 to 5. - [Release notes](https://github.com/stefanzweifel/git-auto-commit-action/releases) - [Changelog](https://github.com/stefanzweifel/git-auto-commit-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/stefanzweifel/git-auto-commit-action/compare/v4...v5) --- updated-dependencies: - dependency-name: stefanzweifel/git-auto-commit-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/publish_lists.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_lists.yml b/.github/workflows/publish_lists.yml index a5f1b0b..920af4d 100644 --- a/.github/workflows/publish_lists.yml +++ b/.github/workflows/publish_lists.yml @@ -44,7 +44,7 @@ jobs: - name: Update documents if: steps.build.outputs.status == 'success' - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@v5 with: # homage to the python linting utility "black" commit_message: ✨🍰✨ From 262a13028f94c1ad2fe10cffa3dca3f0de26e2db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Mar 2024 20:43:21 +0000 Subject: [PATCH 03/14] Bump dependabot/fetch-metadata from 1 to 2 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1 to 2. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/v1...v2) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dependabot_merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependabot_merge.yml b/.github/workflows/dependabot_merge.yml index 4b8899c..7debea7 100644 --- a/.github/workflows/dependabot_merge.yml +++ b/.github/workflows/dependabot_merge.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1 + uses: dependabot/fetch-metadata@v2 with: github-token: ${{ github.token }} From b3035f43f197794d4dcade5dcd424c72399e8084 Mon Sep 17 00:00:00 2001 From: T145 Date: Tue, 2 Jul 2024 21:23:02 -0400 Subject: [PATCH 04/14] fix(v1): Updated naming convention --- dist/{black_nxdomain.txt => BLOCK_NXDOMAIN.txt} | 0 scripts/v1/build_lists.bash | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename dist/{black_nxdomain.txt => BLOCK_NXDOMAIN.txt} (100%) diff --git a/dist/black_nxdomain.txt b/dist/BLOCK_NXDOMAIN.txt similarity index 100% rename from dist/black_nxdomain.txt rename to dist/BLOCK_NXDOMAIN.txt diff --git a/scripts/v1/build_lists.bash b/scripts/v1/build_lists.bash index 566a426..65a1ea0 100755 --- a/scripts/v1/build_lists.bash +++ b/scripts/v1/build_lists.bash @@ -18,14 +18,14 @@ merge_lists() { main() { curl --proto '=https' --tlsv1.3 -H 'Accept: application/vnd.github.v3+json' -sSf https://api.github.com/repos/T145/black-mirror/releases/latest | - jq -r '.assets[] | select(.name | endswith("txt")).browser_download_url' | + jaq -r '.assets[] | select(.name | endswith("txt")).browser_download_url' | aria2c -i- -d ./assets --conf-path='./configs/aria2.conf' local nxlist local list - nxlist='./dist/black_nxdomain.txt' - list='./assets/black_domain.txt' + nxlist='./dist/BLOCK_NXDOMAIN.txt' + list='./assets/BLOCK_DOMAIN.txt' # Max thread count is 204822, as given by `cat /proc/sys/kernel/threads-max` # https://askubuntu.com/questions/1006377/check-the-max-allowed-threads-count-for-sure#1006384 From 97c8ab3db4eac12418f2e7c4d64d25f21365cd85 Mon Sep 17 00:00:00 2001 From: T145 Date: Tue, 2 Jul 2024 21:35:00 -0400 Subject: [PATCH 05/14] fix(v1): Fixed GitHub workflows --- .../{publish_lists.yml => publish.yml} | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) rename .github/workflows/{publish_lists.yml => publish.yml} (70%) diff --git a/.github/workflows/publish_lists.yml b/.github/workflows/publish.yml similarity index 70% rename from .github/workflows/publish_lists.yml rename to .github/workflows/publish.yml index 920af4d..01a06e5 100644 --- a/.github/workflows/publish_lists.yml +++ b/.github/workflows/publish.yml @@ -1,47 +1,48 @@ -name: Publish Lists +--- +name: Publish on: schedule: - cron: '0 6 * * */3' workflow_dispatch: +# https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs +permissions: read-all + jobs: github: - runs-on: ubuntu-latest - + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash container: image: ghcr.io/t145/black-mirror:latest credentials: username: ${{ github.actor }} password: ${{ github.token }} - + options: --user root steps: - name: Checkout repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - name: Generate lists id: build - run: ./scripts/github/workflow.bash - shell: bash - + run: | + chmod -R 755 ./scripts/* + ./scripts/github/workflow.bash - name: Dump output context env: STEPS_CONTEXT: ${{ toJson(steps) }} run: echo "$STEPS_CONTEXT" shell: bash - - name: Create GitHub release if: steps.build.outputs.status == 'success' - uses: marvinpinto/action-automatic-releases@v1.2.1 + uses: marvinpinto/action-automatic-releases@latest with: repo_token: ${{ github.token }} automatic_release_tag: latest prerelease: false title: All Artifacts files: dist/* - - name: Update documents if: steps.build.outputs.status == 'success' uses: stefanzweifel/git-auto-commit-action@v5 From 5e871ccc96faa8197a95220980a987db9f59c411 Mon Sep 17 00:00:00 2001 From: T145 Date: Tue, 2 Jul 2024 21:40:22 -0400 Subject: [PATCH 06/14] fix(v1): Fixed commands --- scripts/v1/build_lists.bash | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/v1/build_lists.bash b/scripts/v1/build_lists.bash index 65a1ea0..521fb9a 100755 --- a/scripts/v1/build_lists.bash +++ b/scripts/v1/build_lists.bash @@ -16,6 +16,19 @@ merge_lists() { sorted "$1" } +# https://github.com/ildar-shaimordanov/perl-utils#sponge +sponge() { + perl -ne ' + push @lines, $_; + END { + open(OUT, ">$file") + or die "sponge: cannot open $file: $!\n"; + print OUT @lines; + close(OUT); + } + ' -s -- -file="$1" +} + main() { curl --proto '=https' --tlsv1.3 -H 'Accept: application/vnd.github.v3+json' -sSf https://api.github.com/repos/T145/black-mirror/releases/latest | jaq -r '.assets[] | select(.name | endswith("txt")).browser_download_url' | From ef27c32f4ca1adb044ac008ab52e12ef56dff347 Mon Sep 17 00:00:00 2001 From: T145 Date: Tue, 2 Jul 2024 23:48:27 -0400 Subject: [PATCH 07/14] fix(v1): More patches --- scripts/v1/build_lists.bash | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/v1/build_lists.bash b/scripts/v1/build_lists.bash index 521fb9a..4174b6e 100755 --- a/scripts/v1/build_lists.bash +++ b/scripts/v1/build_lists.bash @@ -4,18 +4,6 @@ TMP=$(mktemp) readonly TMP trap 'rm -rf "$TMP"' EXIT || exit 1 -# params: file path -sorted() { - parsort -bfiu -S 100% --parallel=200000 -T "$DOWNLOADS" "$1" | sponge "$1" -} - -# merge list 2 into list 1 -# params: list 1, list 2 -merge_lists() { - cat "$1" "$2" >"$1" - sorted "$1" -} - # https://github.com/ildar-shaimordanov/perl-utils#sponge sponge() { perl -ne ' @@ -29,6 +17,18 @@ sponge() { ' -s -- -file="$1" } +# params: file path +sorted() { + parsort -bfiu -S 100% -T "$DOWNLOADS" "$1" | sponge "$1" +} + +# merge list 2 into list 1 +# params: list 1, list 2 +merge_lists() { + cat "$1" "$2" >"$1" + sorted "$1" +} + main() { curl --proto '=https' --tlsv1.3 -H 'Accept: application/vnd.github.v3+json' -sSf https://api.github.com/repos/T145/black-mirror/releases/latest | jaq -r '.assets[] | select(.name | endswith("txt")).browser_download_url' | From 73f9a6b184788b9d7476addf4f77cf209138d310 Mon Sep 17 00:00:00 2001 From: T145 Date: Tue, 2 Jul 2024 23:55:05 -0400 Subject: [PATCH 08/14] fix(v1): Make the commit directory safe --- .github/workflows/publish.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 01a06e5..548a84c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -34,15 +34,8 @@ jobs: STEPS_CONTEXT: ${{ toJson(steps) }} run: echo "$STEPS_CONTEXT" shell: bash - - name: Create GitHub release - if: steps.build.outputs.status == 'success' - uses: marvinpinto/action-automatic-releases@latest - with: - repo_token: ${{ github.token }} - automatic_release_tag: latest - prerelease: false - title: All Artifacts - files: dist/* + - name: Make the working tree safe + run: git config --global --add safe.directory /__w/metalhead/metalhead - name: Update documents if: steps.build.outputs.status == 'success' uses: stefanzweifel/git-auto-commit-action@v5 From e5138aaf6d8b217687e7d93dd3a68a57d0bca03a Mon Sep 17 00:00:00 2001 From: T145 Date: Wed, 3 Jul 2024 00:02:00 -0400 Subject: [PATCH 09/14] fix(v1): More script tuning --- .github/workflows/publish.yml | 2 -- scripts/github/workflow.bash | 14 ++++++-------- scripts/v1/build_lists.bash | 6 +++--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 548a84c..95bd5a5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -34,8 +34,6 @@ jobs: STEPS_CONTEXT: ${{ toJson(steps) }} run: echo "$STEPS_CONTEXT" shell: bash - - name: Make the working tree safe - run: git config --global --add safe.directory /__w/metalhead/metalhead - name: Update documents if: steps.build.outputs.status == 'success' uses: stefanzweifel/git-auto-commit-action@v5 diff --git a/scripts/github/workflow.bash b/scripts/github/workflow.bash index 6dcae75..8938dc5 100755 --- a/scripts/github/workflow.bash +++ b/scripts/github/workflow.bash @@ -1,16 +1,14 @@ #!/usr/bin/env bash main() { - local result - git config --global --add safe.directory /__w/metalhead/metalhead - ./scripts/v1/build_lists.bash - - [[ "$?" = 0 ]] && result='success' || result='failure' - # https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#outputs-for-composite-actions= - # https://help.github.com/en/articles/development-tools-for-github-actions#set-an-output-parameter-set-output - echo "::set-output name=status::${result}" + if ./scripts/v1/build_lists.bash; then + echo "status=success" >>"$GITHUB_OUTPUT" + else + cat <&2 + exit 1 + fi } main diff --git a/scripts/v1/build_lists.bash b/scripts/v1/build_lists.bash index 4174b6e..373ebc5 100755 --- a/scripts/v1/build_lists.bash +++ b/scripts/v1/build_lists.bash @@ -45,14 +45,14 @@ main() { # TODO: Perform these steps for each list! if test -f "$nxlist"; then # TODO: Export JSON from dnsX and use jq to pull out domains & ips - dnsx -r ./configs/resolvers.txt -l "$nxlist" -o "$TMP" -c 200000 -silent -rcode noerror,servfail,refused 1>/dev/null + dnsx -r ./configs/resolvers.txt -l "$nxlist" -o "$TMP" -c 20000 -silent -rcode noerror,servfail,refused 1>/dev/null merge_lists "$list" "$TMP" # nxlist should be small enough that parallel isn't needed grep -Fxvf "$TMP" "$nxlist" | sponge "$nxlist" - dnsx -r ./configs/resolvers.txt -hf "$nxlist" -l "$list" -o "$nxlist" -c 200000 -silent -rcode nxdomain 1>/dev/null + dnsx -r ./configs/resolvers.txt -hf "$nxlist" -l "$list" -o "$nxlist" -c 20000 -silent -rcode nxdomain 1>/dev/null : >"$TMP" else - dnsx -r ./configs/resolvers.txt -l "$list" -o "$nxlist" -c 200000 -silent -rcode nxdomain 1>/dev/null + dnsx -r ./configs/resolvers.txt -l "$list" -o "$nxlist" -c 20000 -silent -rcode nxdomain 1>/dev/null fi rm -rf ./assets/*.txt From 3ada7ec4d7bab6c6bb7ebd525cb1c89424479c90 Mon Sep 17 00:00:00 2001 From: T145 Date: Wed, 3 Jul 2024 00:08:58 -0400 Subject: [PATCH 10/14] fix(v1): Fixed indents --- .editorconfig | 11 ++++++++++- scripts/github/workflow.bash | 2 +- scripts/v1/{build_lists.bash => build.bash} | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 12 deletions(-) rename scripts/v1/{build_lists.bash => build.bash} (88%) mode change 100755 => 100644 diff --git a/.editorconfig b/.editorconfig index 827bef2..78fb9b4 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,7 +7,7 @@ end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true -[*.bash,*.pl] +[*.bash,*.pl,*.awk,*.py] indent_style = space indent_size = 2 @@ -17,3 +17,12 @@ end_of_line = crlf [*.md] max_line_length = off trim_trailing_whitespace = false + +# Ignore binary files +[/bin/**] +charset = unset +end_of_line = unset +insert_final_newline = unset +trim_trailing_whitespace = unset +indent_style = unset +indent_size = unset diff --git a/scripts/github/workflow.bash b/scripts/github/workflow.bash index 8938dc5..749617f 100755 --- a/scripts/github/workflow.bash +++ b/scripts/github/workflow.bash @@ -3,7 +3,7 @@ main() { git config --global --add safe.directory /__w/metalhead/metalhead - if ./scripts/v1/build_lists.bash; then + if ./scripts/v1/build.bash; then echo "status=success" >>"$GITHUB_OUTPUT" else cat <&2 diff --git a/scripts/v1/build_lists.bash b/scripts/v1/build.bash old mode 100755 new mode 100644 similarity index 88% rename from scripts/v1/build_lists.bash rename to scripts/v1/build.bash index 373ebc5..1340d76 --- a/scripts/v1/build_lists.bash +++ b/scripts/v1/build.bash @@ -6,15 +6,15 @@ trap 'rm -rf "$TMP"' EXIT || exit 1 # https://github.com/ildar-shaimordanov/perl-utils#sponge sponge() { - perl -ne ' - push @lines, $_; - END { - open(OUT, ">$file") - or die "sponge: cannot open $file: $!\n"; - print OUT @lines; - close(OUT); - } - ' -s -- -file="$1" + perl -ne ' + push @lines, $_; + END { + open(OUT, ">$file") + or die "sponge: cannot open $file: $!\n"; + print OUT @lines; + close(OUT); + } + ' -s -- -file="$1" } # params: file path @@ -43,7 +43,7 @@ main() { # Max thread count is 204822, as given by `cat /proc/sys/kernel/threads-max` # https://askubuntu.com/questions/1006377/check-the-max-allowed-threads-count-for-sure#1006384 # TODO: Perform these steps for each list! - if test -f "$nxlist"; then + if [ -f "$nxlist" ]; then # TODO: Export JSON from dnsX and use jq to pull out domains & ips dnsx -r ./configs/resolvers.txt -l "$nxlist" -o "$TMP" -c 20000 -silent -rcode noerror,servfail,refused 1>/dev/null merge_lists "$list" "$TMP" From 6a1eec0f15ffdc92222461c12fbaa00e59f4bc0f Mon Sep 17 00:00:00 2001 From: T145 Date: Wed, 3 Jul 2024 00:12:59 -0400 Subject: [PATCH 11/14] fix(v1): Fixed unused variable being included --- scripts/v1/build.bash | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/v1/build.bash b/scripts/v1/build.bash index 1340d76..f6f3ab3 100644 --- a/scripts/v1/build.bash +++ b/scripts/v1/build.bash @@ -19,7 +19,7 @@ sponge() { # params: file path sorted() { - parsort -bfiu -S 100% -T "$DOWNLOADS" "$1" | sponge "$1" + parsort -bfiu -S 100% "$1" | sponge "$1" } # merge list 2 into list 1 @@ -34,6 +34,8 @@ main() { jaq -r '.assets[] | select(.name | endswith("txt")).browser_download_url' | aria2c -i- -d ./assets --conf-path='./configs/aria2.conf' + echo 'Downloaded lists!' + local nxlist local list From 5875e44c366c71ce812685550dcff130ef4f2c6a Mon Sep 17 00:00:00 2001 From: T145 Date: Wed, 3 Jul 2024 00:17:53 -0400 Subject: [PATCH 12/14] fix(v1): Try simple commands in the workflow --- .github/workflows/publish.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 95bd5a5..6dcaeae 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -27,15 +27,13 @@ jobs: - name: Generate lists id: build run: | - chmod -R 755 ./scripts/* - ./scripts/github/workflow.bash - - name: Dump output context - env: - STEPS_CONTEXT: ${{ toJson(steps) }} - run: echo "$STEPS_CONTEXT" - shell: bash + curl --proto '=https' --tlsv1.3 -H 'Accept: application/vnd.github.v3+json' -sSf https://api.github.com/repos/T145/black-mirror/releases/latest | + jaq -r '.assets[] | select(.name | endswith("txt")).browser_download_url' | + aria2c -i- -d ./assets + + dnsx -silent -rcode nxdomain -l './assets/BLOCK_DOMAIN.txt' -o './dist/BLOCK_NXDOMAIN.txt' 1>/dev/null - name: Update documents - if: steps.build.outputs.status == 'success' + if: success() uses: stefanzweifel/git-auto-commit-action@v5 with: # homage to the python linting utility "black" From 151dc4b3384daa8e328d3cf64977a34998830b21 Mon Sep 17 00:00:00 2001 From: T145 Date: Sun, 7 Jul 2024 21:50:28 -0400 Subject: [PATCH 13/14] feat(v2): Run scans in parallel to avoid running actions too long --- .github/ISSUE_TEMPLATE/BUG_REPORT.yml | 52 ++++++++++++ .github/ISSUE_TEMPLATE/DOC_UPDATE.yml | 36 +++++++++ .github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml | 36 +++++++++ .github/ISSUE_TEMPLATE/LIST_MAINTAINENCE.yml | 44 ++++++++++ .github/ISSUE_TEMPLATE/config.yml | 9 +++ .github/dependabot.yml | 3 +- .github/release.yml | 18 +++++ .github/workflows/clean_old_data.yml | 21 +++++ .github/workflows/dependabot_merge.yml | 25 ------ .github/workflows/publish.yml | 29 +++---- .github/README.md => README.md | 0 configs/aria2.conf | 75 ----------------- configs/resolvers.txt | 6 -- scripts/github/workflow.bash | 17 ++-- scripts/v1/build.bash | 63 --------------- scripts/v2/publish.bash | 85 ++++++++++++++++++++ 16 files changed, 329 insertions(+), 190 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/BUG_REPORT.yml create mode 100644 .github/ISSUE_TEMPLATE/DOC_UPDATE.yml create mode 100644 .github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml create mode 100644 .github/ISSUE_TEMPLATE/LIST_MAINTAINENCE.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/release.yml create mode 100644 .github/workflows/clean_old_data.yml delete mode 100644 .github/workflows/dependabot_merge.yml rename .github/README.md => README.md (100%) delete mode 100644 configs/aria2.conf delete mode 100644 configs/resolvers.txt delete mode 100644 scripts/v1/build.bash create mode 100644 scripts/v2/publish.bash diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.yml b/.github/ISSUE_TEMPLATE/BUG_REPORT.yml new file mode 100644 index 0000000..be4e52b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.yml @@ -0,0 +1,52 @@ +--- +name: πŸ› Bug Report +description: File a bug report. +title: "[bug]: " +labels: ["bug"] +assignees: + - T145 +body: + - type: markdown + attributes: + value: Thanks for taking the time to fill out this form! + - type: input + id: contact + attributes: + label: Contact Details + description: How can we get in touch with you if we need more info? + placeholder: ex. email@example.com + validations: + required: false + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Also tell us, what did you expect to happen? + placeholder: Tell us what you see! + value: "A bug happened!" + validations: + required: true + - type: input + id: operating-system + attributes: + label: Operating System + description: Which operating system are you running the scripts on? + placeholder: ex. Ubuntu 21.04 + validations: + required: true + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: bash + validations: + required: true + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/T145/black-mirror/blob/master/CODE_OF_CONDUCT.md#contributor-covenant-code-of-conduct) + options: + - label: I agree to follow this project's Code of Conduct + required: true diff --git a/.github/ISSUE_TEMPLATE/DOC_UPDATE.yml b/.github/ISSUE_TEMPLATE/DOC_UPDATE.yml new file mode 100644 index 0000000..747445e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/DOC_UPDATE.yml @@ -0,0 +1,36 @@ +--- +name: ✍🏿 Documentation Update +description: Grammar fixes, broken links, outdated information, etc. +title: "[docs]: " +labels: ["documentation"] +assignees: + - T145 +body: + - type: markdown + attributes: + value: Thanks for taking the time to fill out this form! + - type: input + id: contact + attributes: + label: Contact Details + description: How can we get in touch with you if we need more info? + placeholder: ex. email@example.com + validations: + required: false + - type: textarea + id: request + attributes: + label: What needs a change? + description: Also tell us implementation details if applicable. + placeholder: Place details here! + value: "This needed a grammar fix..." + validations: + required: true + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/T145/black-mirror/blob/master/CODE_OF_CONDUCT.md#contributor-covenant-code-of-conduct) + options: + - label: I agree to follow this project's Code of Conduct + required: true diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml new file mode 100644 index 0000000..c2b97d9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml @@ -0,0 +1,36 @@ +--- +name: 🌟 Feature Request +description: Request a new feature. +title: "[feature]: " +labels: ["enhancement"] +assignees: + - T145 +body: + - type: markdown + attributes: + value: Thanks for taking the time to fill out this form! + - type: input + id: contact + attributes: + label: Contact Details + description: How can we get in touch with you if we need more info? + placeholder: ex. email@example.com + validations: + required: false + - type: textarea + id: request + attributes: + label: What's your idea? + description: Also tell us implementation details if applicable. + placeholder: Place details here! + value: "This project could really use..." + validations: + required: true + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/T145/black-mirror/blob/master/CODE_OF_CONDUCT.md#contributor-covenant-code-of-conduct) + options: + - label: I agree to follow this project's Code of Conduct + required: true diff --git a/.github/ISSUE_TEMPLATE/LIST_MAINTAINENCE.yml b/.github/ISSUE_TEMPLATE/LIST_MAINTAINENCE.yml new file mode 100644 index 0000000..0716e0b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/LIST_MAINTAINENCE.yml @@ -0,0 +1,44 @@ +--- +name: πŸ“š List Maintenance +description: Adding a new list source, manifesto violations, etc. +title: "[list]: " +labels: ["maintenance"] +assignees: + - T145 +body: + - type: markdown + attributes: + value: Thanks for taking the time to fill out this form! + - type: input + id: contact + attributes: + label: Contact Details + description: How can we get in touch with you if we need more info? + placeholder: ex. email@example.com + validations: + required: false + - type: textarea + id: request + attributes: + label: What needs to change? + description: Anything related to the lists, from host redundancies to new list recommendations. + placeholder: Details go here! + value: "Please change this:" + validations: + required: true + - type: input + id: operating-system + attributes: + label: Operating System + description: If relevant, please include the OS or platform you're using the list(s) on. + placeholder: ex. Ubuntu 21.04 + validations: + required: false + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/T145/black-mirror/blob/master/CODE_OF_CONDUCT.md#contributor-covenant-code-of-conduct) + options: + - label: I agree to follow this project's Code of Conduct + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..5b1a4a3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,9 @@ +--- +blank_issues_enabled: false +contact_links: + - name: GitHub Community Support + url: https://github.com/orgs/community/discussions + about: Please ask and answer questions here. + - name: GitHub Security Bug Bounty + url: https://bounty.github.com/ + about: Please report security vulnerabilities here. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3bd824c..7c804d8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,9 +1,10 @@ # Set update schedule for GitHub Actions # https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/keeping-your-actions-up-to-date-with-dependabot +--- version: 2 updates: - package-ecosystem: "github-actions" - directory: "/" + directory: ".github/workflows" schedule: interval: "daily" diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..f8f2f95 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,18 @@ +ο»Ώ--- +# https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes +changelog: + exclude: + labels: + - ignore-for-release + authors: + - octocat + categories: + - title: Features + labels: + - "*" + - title: Bug Fixes + labels: + - fix + - title: Continuous Integration + labels: + - ci diff --git a/.github/workflows/clean_old_data.yml b/.github/workflows/clean_old_data.yml new file mode 100644 index 0000000..5a68b7b --- /dev/null +++ b/.github/workflows/clean_old_data.yml @@ -0,0 +1,21 @@ +--- +name: Clean Old Data + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + +permissions: read-all + +jobs: + clean: + runs-on: ubuntu-22.04 + steps: + # https://github.com/Mattraks/delete-workflow-runs + - name: Remove expired workflow runs + uses: Mattraks/delete-workflow-runs@c3872d103f18a947d291cd2c3c81c90397b4b830 + with: + token: ${{ secrets.FOR_WEBHOOKS_SECRET }} + retain_days: 1 + keep_minimum_runs: 2 diff --git a/.github/workflows/dependabot_merge.yml b/.github/workflows/dependabot_merge.yml deleted file mode 100644 index 7debea7..0000000 --- a/.github/workflows/dependabot_merge.yml +++ /dev/null @@ -1,25 +0,0 @@ -# https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request -name: Dependabot Auto Merge - -on: pull_request - -permissions: - pull-requests: write - contents: write - -jobs: - dependabot: - runs-on: ubuntu-latest - if: ${{ github.actor == 'dependabot[bot]' }} - steps: - - name: Dependabot metadata - id: metadata - uses: dependabot/fetch-metadata@v2 - with: - github-token: ${{ github.token }} - - - name: Enable auto-merge for Dependabot PRs - run: gh pr merge --auto --merge "$PR_URL" - env: - PR_URL: ${{ github.event.pull_request.html_url }} - GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6dcaeae..5f36f06 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,20 +1,21 @@ --- -name: Publish +name: Publish List on: schedule: - cron: '0 6 * * */3' workflow_dispatch: -# https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs permissions: read-all jobs: - github: + publish: runs-on: ubuntu-22.04 defaults: run: shell: bash + permissions: + contents: write container: image: ghcr.io/t145/black-mirror:latest credentials: @@ -24,17 +25,17 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - - name: Generate lists + - name: Publish List id: build run: | - curl --proto '=https' --tlsv1.3 -H 'Accept: application/vnd.github.v3+json' -sSf https://api.github.com/repos/T145/black-mirror/releases/latest | - jaq -r '.assets[] | select(.name | endswith("txt")).browser_download_url' | - aria2c -i- -d ./assets - - dnsx -silent -rcode nxdomain -l './assets/BLOCK_DOMAIN.txt' -o './dist/BLOCK_NXDOMAIN.txt' 1>/dev/null - - name: Update documents - if: success() - uses: stefanzweifel/git-auto-commit-action@v5 + chmod -R 755 ./scripts/* + ./scripts/github/workflow.bash + - name: Dump output context + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + run: echo "$STEPS_CONTEXT" + - name: Commit changes + if: steps.build.outputs.status == 'success' + uses: stefanzweifel/git-auto-commit-action@7d0ca8f849305c56d36f992cfb1519b1af5b33f8 with: - # homage to the python linting utility "black" - commit_message: ✨🍰✨ + commit_message: "ci(build): ✨🍰✨" diff --git a/.github/README.md b/README.md similarity index 100% rename from .github/README.md rename to README.md diff --git a/configs/aria2.conf b/configs/aria2.conf deleted file mode 100644 index 89346b2..0000000 --- a/configs/aria2.conf +++ /dev/null @@ -1,75 +0,0 @@ -############# -# Generic Settings -############# - -quiet=true -enable-color=false - -# Disable unnecessary features -no-netrc=true -realtime-chunk-checksum=false - -############# -# File Allocation -############# - -auto-file-renaming=false -content-disposition-default-utf8=true -disk-cache=64M -file-allocation=falloc -no-file-allocation-limit=8M -allow-overwrite=true -allow-piece-length-change=true -enable-mmap=true - -############# -# Logging -############# - -console-log-level=error -log-level=notice - -############# -# Connection Settings -############# - -remote-time=true -conditional-get=true -user-agent=Wget/1.21 -http-no-cache=true -http-accept-gzip=true - -# Unlimited download settings -max-overall-download-limit=0 -#max-connection-per-server=8 -max-connection-per-server=16 -split=32 -max-concurrent-downloads=50 -min-split-size=8M - -# Most stealth-mode services usually cap connections at three: -# if hardened servers kick us off use the settings below -#max-connection-per-server=3 -#max-concurrent-downloads=20 - -# https://aria2.github.io/manual/en/html/aria2c.html#cmdoption-optimize-concurrent-downloads -# Tweaking A & B in OCD may yield faster downloads -optimize-concurrent-downloads=true - -# AAAA record lookups are slow -disable-ipv6=true -# DNS providers in paired usage order: -## Quad9 Unsecured (https://www.quad9.net/service/service-addresses-and-features#unsec) -## DNSWatch (https://dns.watch/) -## OpenDNS (https://www.opendns.com/) -async-dns-server=9.9.9.10:53,149.112.112.10:53,84.200.69.80:53,84.200.70.40:53,208.67.222.222:53,208.67.220.220:53 - -############# -# Error Handling -############# - -timeout=60 -connect-timeout=30 -max-tries=5 -retry-wait=10 -max-file-not-found=1 diff --git a/configs/resolvers.txt b/configs/resolvers.txt deleted file mode 100644 index 9a24f33..0000000 --- a/configs/resolvers.txt +++ /dev/null @@ -1,6 +0,0 @@ -9.9.9.10:53 -149.112.112.10:53 -84.200.69.80:53 -84.200.70.40:53 -208.67.222.222:53 -208.67.220.220:53 diff --git a/scripts/github/workflow.bash b/scripts/github/workflow.bash index 749617f..e6ba3d4 100755 --- a/scripts/github/workflow.bash +++ b/scripts/github/workflow.bash @@ -1,14 +1,19 @@ #!/usr/bin/env bash +# Handles everything involved in GitHub CI processing main() { - git config --global --add safe.directory /__w/metalhead/metalhead + git config --global --add safe.directory /__w/metalhead/metalhead - if ./scripts/v1/build.bash; then - echo "status=success" >>"$GITHUB_OUTPUT" - else - cat <&2 + # https://stackoverflow.com/questions/4336035/performance-profiling-tools-for-shell-scripts + if PS4='+ $(date "+%s.%N ($LINENO) ")' ./scripts/v2/publish.bash; then + # https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#outputs-for-composite-actions= + # https://help.github.com/en/articles/development-tools-for-github-actions#set-an-output-parameter-set-output + # https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ + echo "status=success" >>"$GITHUB_OUTPUT" + else + cat <&2 exit 1 - fi + fi } main diff --git a/scripts/v1/build.bash b/scripts/v1/build.bash deleted file mode 100644 index f6f3ab3..0000000 --- a/scripts/v1/build.bash +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -TMP=$(mktemp) -readonly TMP -trap 'rm -rf "$TMP"' EXIT || exit 1 - -# https://github.com/ildar-shaimordanov/perl-utils#sponge -sponge() { - perl -ne ' - push @lines, $_; - END { - open(OUT, ">$file") - or die "sponge: cannot open $file: $!\n"; - print OUT @lines; - close(OUT); - } - ' -s -- -file="$1" -} - -# params: file path -sorted() { - parsort -bfiu -S 100% "$1" | sponge "$1" -} - -# merge list 2 into list 1 -# params: list 1, list 2 -merge_lists() { - cat "$1" "$2" >"$1" - sorted "$1" -} - -main() { - curl --proto '=https' --tlsv1.3 -H 'Accept: application/vnd.github.v3+json' -sSf https://api.github.com/repos/T145/black-mirror/releases/latest | - jaq -r '.assets[] | select(.name | endswith("txt")).browser_download_url' | - aria2c -i- -d ./assets --conf-path='./configs/aria2.conf' - - echo 'Downloaded lists!' - - local nxlist - local list - - nxlist='./dist/BLOCK_NXDOMAIN.txt' - list='./assets/BLOCK_DOMAIN.txt' - - # Max thread count is 204822, as given by `cat /proc/sys/kernel/threads-max` - # https://askubuntu.com/questions/1006377/check-the-max-allowed-threads-count-for-sure#1006384 - # TODO: Perform these steps for each list! - if [ -f "$nxlist" ]; then - # TODO: Export JSON from dnsX and use jq to pull out domains & ips - dnsx -r ./configs/resolvers.txt -l "$nxlist" -o "$TMP" -c 20000 -silent -rcode noerror,servfail,refused 1>/dev/null - merge_lists "$list" "$TMP" - # nxlist should be small enough that parallel isn't needed - grep -Fxvf "$TMP" "$nxlist" | sponge "$nxlist" - dnsx -r ./configs/resolvers.txt -hf "$nxlist" -l "$list" -o "$nxlist" -c 20000 -silent -rcode nxdomain 1>/dev/null - : >"$TMP" - else - dnsx -r ./configs/resolvers.txt -l "$list" -o "$nxlist" -c 20000 -silent -rcode nxdomain 1>/dev/null - fi - - rm -rf ./assets/*.txt -} - -main diff --git a/scripts/v2/publish.bash b/scripts/v2/publish.bash new file mode 100644 index 0000000..36d86d1 --- /dev/null +++ b/scripts/v2/publish.bash @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +#shopt -s extdebug # or --debugging +set +H +o history # disable history features (helps avoid errors from "!" in strings) +shopt -u cmdhist # would be enabled and have no effect otherwise +shopt -s execfail # ensure interactive and non-interactive runtime are similar +shopt -s extglob # enable extended pattern matching (https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html) +set -euET -o pipefail # put bash into "strict mode" & have it give descriptive errors +umask 055 # change all generated file permissions from 755 to 700 + +OUTDIR='assets' +DOWNLOADS=$(mktemp -d) +TMP=$(mktemp) +ERROR_LOG='logs/error.log' +JOB_LOG='logs/jobs.log' +NXLIST='./dist/BLOCK_NXDOMAIN.txt' +TARGET='./assets/BLOCK_DOMAIN.txt' +readonly OUTDIR DOWNLOADS TMP ERROR_LOG JOB_LOG + +# https://github.com/ildar-shaimordanov/perl-utils#sponge +sponge() { + perl5.41.1 -ne ' + push @lines, $_; + END { + open(OUT, ">$file") + or die "sponge: cannot open $file: $!\n"; + print OUT @lines; + close(OUT); + } + ' -s -- -file="$1" +} + +# params: file to sort, +sorted() { + parsort -bfiu -S 100% -T "$DOWNLOADS" "$1" | sponge "$1" + echo "[INFO] Organized: ${1}" +} + +# merge list 2 into list 1 +# params: list 1, list 2 +# merge_lists() { +# cat "$1" "$2" >"$1" +# sorted "$1" +# } + +main() { + trap 'rm -rf "$TMP"' EXIT || exit 1 + mkdir -p "$OUTDIR" + # clear all logs + #find -P -O3 ./logs -depth -type f -print0 | xargs -0 truncate -s 0 + chmod -t /tmp + + curl --proto '=https' --tlsv1.3 -H 'Accept: application/vnd.github.v3+json' -sSf https://api.github.com/repos/T145/black-mirror/releases/latest | + jaq -r '.assets[] | select(.name | startswith("BLOCK_DOMAIN")) | select(.name | endswith(".txt")).browser_download_url' | + aria2c -i- -d "$OUTDIR" + + split -d -l 500000 --additional-suffix .txt "$TARGET" "${OUTDIR}/BLOCK_DOMAIN_" + rm "$TARGET" + find -P -O3 "$OUTDIR" -type f -name *.txt -exec sem dnsx -l {} -o "{}2" -silent -rcode nxdomain 1>/dev/null \; + sem --wait + + # Max thread count is 204822, as given by `cat /proc/sys/kernel/threads-max` + # https://askubuntu.com/questions/1006377/check-the-max-allowed-threads-count-for-sure#1006384 + # TODO: Perform these steps for each list! + # if [ -f "$NXLIST" ]; then + # # TODO: Export JSON from dnsX and use jq to pull out domains & ips + # dnsx -r ./configs/resolvers.txt -l "$NXLIST" -o "$TMP" -c 20000 -silent -rcode noerror,servfail,refused 1>/dev/null + # merge_lists "$TARGET" "$TMP" + # # nxlist should be small enough that parallel isn't needed + # grep -Fxvf "$TMP" "$NXLIST" | sponge "$NXLIST" + # dnsx -r ./configs/resolvers.txt -hf "$NXLIST" -l "$TARGET" -o "$NXLIST" -c 20000 -silent -rcode nxdomain 1>/dev/null + # : >"$TMP" + # else + # dnsx -r ./configs/resolvers.txt -l "$TARGET" -o "$NXLIST" -c 20000 -silent -rcode nxdomain 1>/dev/null + # fi + + mawk '{print $1}' "${OUTDIR}/BLOCK_DOMAIN_*.txt2" | sponge "$NXLIST" + sorted "$NXLIST" + rm -rf "$OUTDIR" + + chmod +t /tmp +} + +# https://github.com/koalaman/shellcheck/wiki/SC2218 +main From 0e0e49ee89887f9593d169ad78a4a5dcd2ffc369 Mon Sep 17 00:00:00 2001 From: T145 Date: Sun, 7 Jul 2024 21:51:18 -0400 Subject: [PATCH 14/14] fix(v2): Updated README --- README.md => .github/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README.md => .github/README.md (100%) diff --git a/README.md b/.github/README.md similarity index 100% rename from README.md rename to .github/README.md