From e419a284d8df779c26294ba4553637ab9e302489 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 00:06:30 +0800 Subject: [PATCH 01/12] Added workflow file --- ..._verify_file_does_not_end_with_newline.yml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/wordlist-updater_verify_file_does_not_end_with_newline.yml diff --git a/.github/workflows/wordlist-updater_verify_file_does_not_end_with_newline.yml b/.github/workflows/wordlist-updater_verify_file_does_not_end_with_newline.yml new file mode 100644 index 00000000..5b53b4c8 --- /dev/null +++ b/.github/workflows/wordlist-updater_verify_file_does_not_end_with_newline.yml @@ -0,0 +1,27 @@ +# Source: https://github.com/danielmiessler/SecLists/blob/master/.github/workflows/wordlist-validator_verify_entries_for_starting_with_slash.yml + +name: Wordlist Validator - Verify if any file end with a new line +on: + push: + paths: + - "**.txt" + pull_request: + paths: + - "**.txt" + workflow_dispatch: + +jobs: + check_files_changed: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Get changed files + id: changed-files + with: + files: "**/*.txt" + uses: tj-actions/changed-files@v34 + - name: Analyze all added or modified files + run: | + ./.bin/new-line-checker.py "${{ steps.changed-files.outputs.all_changed_files }}" \ No newline at end of file From 61b4435381a19b2c22bab424d76866232a40e1ca Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 00:06:46 +0800 Subject: [PATCH 02/12] imported file from fire-av --- .bin/new-line-checker.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 .bin/new-line-checker.py diff --git a/.bin/new-line-checker.py b/.bin/new-line-checker.py new file mode 100755 index 00000000..2258454e --- /dev/null +++ b/.bin/new-line-checker.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +# test string: ./.bin/new-line-checker.py "Fuzzing/file-extensions-all-cases.txt Fuzzing/file-extensions-lower-case.txt Fuzzing/file-extensions-upper-case.txt Fuzzing/file-extensions.txt" + +import os + +print("[+] New line check") + +files = [] +dirs = ['sources/','blacklists/','whitelists/'] + +for i in dirs: + for root,_,file_list in os.walk(i): + for file in file_list: + files.append(os.path.join(root,file)) + +for i in files: + f=open(i,"r") + contents=f.read() + + if len(contents) == 0: + continue + + if contents[-1] == '\n': + print("[!] %s ends with a new line"%(i)) + exit(2) + # print("[+] %s passed new line check"%(i)) + +print("[+] All files passed checks") +exit(0) \ No newline at end of file From f794f2ff83ccd88cc96463d66a9ce46d90ad0497 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 00:12:09 +0800 Subject: [PATCH 03/12] added new line checker for all push --- .bin/new-line-checker.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.bin/new-line-checker.py b/.bin/new-line-checker.py index 2258454e..c7716327 100755 --- a/.bin/new-line-checker.py +++ b/.bin/new-line-checker.py @@ -3,16 +3,16 @@ # test string: ./.bin/new-line-checker.py "Fuzzing/file-extensions-all-cases.txt Fuzzing/file-extensions-lower-case.txt Fuzzing/file-extensions-upper-case.txt Fuzzing/file-extensions.txt" import os +import sys print("[+] New line check") -files = [] -dirs = ['sources/','blacklists/','whitelists/'] +files=sys.argv[1].split(" ") -for i in dirs: - for root,_,file_list in os.walk(i): - for file in file_list: - files.append(os.path.join(root,file)) +for i in files: + if not os.path.isfile(i): + print("[!] %s does not exist!"%(i)) + exit(2) for i in files: f=open(i,"r") @@ -24,7 +24,7 @@ for i in files: if contents[-1] == '\n': print("[!] %s ends with a new line"%(i)) exit(2) - # print("[+] %s passed new line check"%(i)) + print("[+] %s passed new line check"%(i)) print("[+] All files passed checks") -exit(0) \ No newline at end of file +# exit(0) \ No newline at end of file From 066ede435628cdb24cb38ba347030053887b0a00 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 00:25:59 +0800 Subject: [PATCH 04/12] Added new line and empty line checker --- ...hecker.py => new-line-and-empty-line-checker.py} | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) rename .bin/{new-line-checker.py => new-line-and-empty-line-checker.py} (66%) diff --git a/.bin/new-line-checker.py b/.bin/new-line-and-empty-line-checker.py similarity index 66% rename from .bin/new-line-checker.py rename to .bin/new-line-and-empty-line-checker.py index c7716327..9fcd71b5 100755 --- a/.bin/new-line-checker.py +++ b/.bin/new-line-and-empty-line-checker.py @@ -17,14 +17,17 @@ for i in files: for i in files: f=open(i,"r") contents=f.read() - - if len(contents) == 0: - continue if contents[-1] == '\n': - print("[!] %s ends with a new line"%(i)) + print("[!] %s ends with a new line!"%(i)) exit(2) - print("[+] %s passed new line check"%(i)) + print("[+] %s passed new line check!"%(i)) + + for j in contents.split('\n'): + if len(i)==0: + print("[!] %s has an empty entry!"%(i)) + exit(2) + print("[+] %s passed empty line check!"%(i)) print("[+] All files passed checks") # exit(0) \ No newline at end of file From 6f4b5e061def60717af7e08ec7ff1f15a48704f3 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 00:26:07 +0800 Subject: [PATCH 05/12] created wrapper script --- .bin/validators.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 .bin/validators.sh diff --git a/.bin/validators.sh b/.bin/validators.sh new file mode 100755 index 00000000..bf2e3f18 --- /dev/null +++ b/.bin/validators.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# https://stackoverflow.com/questions/3822621/how-to-exit-if-a-command-failed + +set -e +set -o pipefail + +# wrapper for all the checking scripts +echo $1 +./.bin/check-file-for-starting-slash "$1" +./.bin/new-line-and-empty-line-checker.py "$1" \ No newline at end of file From 96a9fbf3d3dcf445ce0176c87cbaff3e9bdc81e4 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 00:27:37 +0800 Subject: [PATCH 06/12] removed that --- ..._verify_file_does_not_end_with_newline.yml | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 .github/workflows/wordlist-updater_verify_file_does_not_end_with_newline.yml diff --git a/.github/workflows/wordlist-updater_verify_file_does_not_end_with_newline.yml b/.github/workflows/wordlist-updater_verify_file_does_not_end_with_newline.yml deleted file mode 100644 index 5b53b4c8..00000000 --- a/.github/workflows/wordlist-updater_verify_file_does_not_end_with_newline.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Source: https://github.com/danielmiessler/SecLists/blob/master/.github/workflows/wordlist-validator_verify_entries_for_starting_with_slash.yml - -name: Wordlist Validator - Verify if any file end with a new line -on: - push: - paths: - - "**.txt" - pull_request: - paths: - - "**.txt" - workflow_dispatch: - -jobs: - check_files_changed: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Get changed files - id: changed-files - with: - files: "**/*.txt" - uses: tj-actions/changed-files@v34 - - name: Analyze all added or modified files - run: | - ./.bin/new-line-checker.py "${{ steps.changed-files.outputs.all_changed_files }}" \ No newline at end of file From a4df7ab34a54e21503a76dfea486fc0dd5ac84d2 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 00:28:39 +0800 Subject: [PATCH 07/12] rename and added modification notes --- ...for_starting_with_slash.yml => wordlist-validator.yml} | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) rename .github/workflows/{wordlist-validator_verify_entries_for_starting_with_slash.yml => wordlist-validator.yml} (72%) diff --git a/.github/workflows/wordlist-validator_verify_entries_for_starting_with_slash.yml b/.github/workflows/wordlist-validator.yml similarity index 72% rename from .github/workflows/wordlist-validator_verify_entries_for_starting_with_slash.yml rename to .github/workflows/wordlist-validator.yml index a40be7cd..1adc2a67 100644 --- a/.github/workflows/wordlist-validator_verify_entries_for_starting_with_slash.yml +++ b/.github/workflows/wordlist-validator.yml @@ -2,8 +2,10 @@ # Sources: # https://dev.to/scienta/get-changed-files-in-github-actions-1p36 # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions -# https://github.com/marketplace/actions/changed-files -name: Wordlist Validator - Verify if any file entry start with a slash +# https://github.com/marketplace/actions/changed-files\ +# +# Modified to add a validator script +name: Wordlist Validator - Run a validator script to prevent dangerous pushes on: push: paths: @@ -26,4 +28,4 @@ jobs: uses: tj-actions/changed-files@v34 - name: Analyze all added or modified files run: | - ./.bin/check-file-for-starting-slash "${{ steps.changed-files.outputs.all_changed_files }}" \ No newline at end of file + ./.bin/validators.sh "${{ steps.changed-files.outputs.all_changed_files }}" \ No newline at end of file From a3ef749d1d724a78633a8e423004a12b4cc9f2c6 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 00:32:04 +0800 Subject: [PATCH 08/12] added code that just exit if no args passed --- .bin/new-line-and-empty-line-checker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.bin/new-line-and-empty-line-checker.py b/.bin/new-line-and-empty-line-checker.py index 9fcd71b5..9b134425 100755 --- a/.bin/new-line-and-empty-line-checker.py +++ b/.bin/new-line-and-empty-line-checker.py @@ -7,6 +7,9 @@ import sys print("[+] New line check") +if not sys.argv[1]: + exit(0) + files=sys.argv[1].split(" ") for i in files: From c9d4fe0853408b31021efbeba7a899f07c138b45 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 00:33:17 +0800 Subject: [PATCH 09/12] changed description to accurately reflect purpose --- .github/workflows/wordlist-validator.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wordlist-validator.yml b/.github/workflows/wordlist-validator.yml index 1adc2a67..15d2a968 100644 --- a/.github/workflows/wordlist-validator.yml +++ b/.github/workflows/wordlist-validator.yml @@ -5,7 +5,7 @@ # https://github.com/marketplace/actions/changed-files\ # # Modified to add a validator script -name: Wordlist Validator - Run a validator script to prevent dangerous pushes +name: Wordlist Validator - Run a validator script to check for dangerous pushes on: push: paths: From 86bebe95c492a8dcb9183cff7fa2759a2d26fc71 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 00:41:40 +0800 Subject: [PATCH 10/12] Damn typo! --- .bin/new-line-and-empty-line-checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.bin/new-line-and-empty-line-checker.py b/.bin/new-line-and-empty-line-checker.py index 9b134425..ba6c97a4 100755 --- a/.bin/new-line-and-empty-line-checker.py +++ b/.bin/new-line-and-empty-line-checker.py @@ -26,11 +26,11 @@ for i in files: exit(2) print("[+] %s passed new line check!"%(i)) - for j in contents.split('\n'): - if len(i)==0: + for line in contents.split('\n'): + if len(line)==0: print("[!] %s has an empty entry!"%(i)) exit(2) print("[+] %s passed empty line check!"%(i)) print("[+] All files passed checks") -# exit(0) \ No newline at end of file +# exit(0) From 3da34d777ebcf45ca9ac15e76c953e83950c52a5 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 00:50:46 +0800 Subject: [PATCH 11/12] Grammar fix --- .github/workflows/wordlist-validator.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wordlist-validator.yml b/.github/workflows/wordlist-validator.yml index 15d2a968..80a94422 100644 --- a/.github/workflows/wordlist-validator.yml +++ b/.github/workflows/wordlist-validator.yml @@ -5,7 +5,7 @@ # https://github.com/marketplace/actions/changed-files\ # # Modified to add a validator script -name: Wordlist Validator - Run a validator script to check for dangerous pushes +name: Wordlist Validator - Runs a validator script to check for dangerous pushes on: push: paths: @@ -28,4 +28,4 @@ jobs: uses: tj-actions/changed-files@v34 - name: Analyze all added or modified files run: | - ./.bin/validators.sh "${{ steps.changed-files.outputs.all_changed_files }}" \ No newline at end of file + ./.bin/validators.sh "${{ steps.changed-files.outputs.all_changed_files }}" From 5cdf3deb4c7f3167bdd98fa90e7ac807b87e22fe Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Sat, 25 Nov 2023 01:34:28 +0800 Subject: [PATCH 12/12] more verbose logging --- .bin/new-line-and-empty-line-checker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.bin/new-line-and-empty-line-checker.py b/.bin/new-line-and-empty-line-checker.py index ba6c97a4..2818198c 100755 --- a/.bin/new-line-and-empty-line-checker.py +++ b/.bin/new-line-and-empty-line-checker.py @@ -26,10 +26,13 @@ for i in files: exit(2) print("[+] %s passed new line check!"%(i)) + counter=1 + for line in contents.split('\n'): if len(line)==0: - print("[!] %s has an empty entry!"%(i)) + print("[!] %s has an empty entry at line %i!"%(i,counter)) exit(2) + counter+=1 print("[+] %s passed empty line check!"%(i)) print("[+] All files passed checks")