diff --git a/.bin/brute-force-renormalize.sh b/.bin/brute-force-renormalize.sh new file mode 100644 index 00000000..ad56642f --- /dev/null +++ b/.bin/brute-force-renormalize.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +FILE="$1" + +if [[ -z "$FILE" ]]; then + echo "This script renormalizes a file until its SHA256 hash stops changing, ensuring proper normalization." + echo "Usage: $0 " + exit 1 +fi + +while true; do + ORIGINAL_HASH=$(sha256sum "$FILE" | awk '{print $1}') + + git add --renormalize "$FILE" && git add "$FILE" + + NEW_HASH=$(sha256sum "$FILE" | awk '{print $1}') + + if [[ "$ORIGINAL_HASH" == "$NEW_HASH" ]]; then + break + fi +done diff --git a/.github/workflows/remote-wordlists-updater.yml b/.github/workflows/remote-wordlists-updater.yml index aaa7210a..31cefc37 100644 --- a/.github/workflows/remote-wordlists-updater.yml +++ b/.github/workflows/remote-wordlists-updater.yml @@ -17,6 +17,12 @@ jobs: - name: Update lists run: ./.bin/wordlist-updaters/updater.py + - name: Ensure UTF-8 encoding + run: | + wget https://raw.githubusercontent.com/ItsIgnacioPortal/utf8fixer/f42bfe64db0bc095e9e93a10ac6bba746f09670c/utf8fixer.py + chmod +x utf8fixer.py + ./utf8fixer.py Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt utf8 + - name: Commit files if changed run: | git add -N . @@ -26,6 +32,8 @@ jobs: else echo "[+] Files were changed! Pushing changed..." git add --renormalize -A && git add -A + chmod +x ./.bin/brute-force-renormalize.sh + ./.bin/brute-force-renormalize.sh ./Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --local user.name "github-actions[bot]"