30 lines
1.3 KiB
Bash
Executable File
30 lines
1.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Script to build a list of of environment identifiers based on sub domain names.
|
|
## References:
|
|
# See https://blog.majestic.com/development/alexa-top-1-million-sites-retired-heres-majestic-million/
|
|
# See https://github.com/danielmiessler/SecLists/issues/654
|
|
# See https://github.com/danielmiessler/SecLists/pull/671
|
|
## Requires jq
|
|
#
|
|
# Add more top level domain in the expression below
|
|
TLD_WANTED_EXPR="\.(lu|be)$"
|
|
#
|
|
echo "[+] Download Majestic CSV file..."
|
|
wget -O majestic.csv https://downloads.majestic.com/majestic_million.csv
|
|
echo "[+] Extract wanted domains..."
|
|
cat majestic.csv | cut -d',' -f3 | grep -E $TLD_WANTED_EXPR > domains.txt
|
|
wc -l domains.txt
|
|
echo "[+] Extract sub domains via Certificate Transparency logs (https://crt.sh)..."
|
|
while IFS= read -r line
|
|
do
|
|
printf "\rDomain: %-40s" "$line"
|
|
curl -sk "https://crt.sh/?q=$line&output=json" | jq -r ".[].name_value" | cut -d'.' -f1 1>> subdomains.txt 2>/dev/null
|
|
done < domains.txt
|
|
cat subdomains.txt | sort -u > subdomains.tmp
|
|
mv subdomains.tmp subdomains.txt
|
|
wc -l subdomains.txt
|
|
echo "[+] Extract environment like sub domains..."
|
|
grep -Ei "^(de|dv|ts|te|in|st|ho|pr|pp)" subdomains.txt > env-like-subdomains.txt
|
|
echo "[i] Manually review the generated file 'env-like-subdomains.txt' for accurate content."
|
|
wc -l env-like-subdomains.txt
|