sync/SecLists
1
0
Fork 0
mirror of https://github.com/danielmiessler/SecLists.git synced 2025-05-02 03:26:42 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #1064 from molangning/patch-update-frequency

Browse source 
Patch update frequency and script
This commit is contained in:
g0tmi1k 2024-08-12 20:30:53 +01:00 • committed by GitHub
commit f54b4ad5fd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 68 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.bin/wordlist-updaters/sources.json
View file

@ -5,7 +5,7 @@
"source": "https://raw.githubusercontent.com/wallarm/jwt-secrets/master/jwt.secrets.list", "source": "https://raw.githubusercontent.com/wallarm/jwt-secrets/master/jwt.secrets.list",
"output": "Passwords/scraped-JWT-secrets.txt", "output": "Passwords/scraped-JWT-secrets.txt",
"post_run_script": "", "post_run_script": "",
"frequency": "6h" "frequency": "3d"
}, },
{ {
"name": "Trickest wordlist update", "name": "Trickest wordlist update",

89
.bin/wordlist-updaters/updater.py
View file

@ -17,20 +17,21 @@ FREQUENCY_REGEX = r"^(?:([0-9]+)d|())(?:([0-9]+)h|())(?!.*?d)$"
VALID_TYPES = ["file", "git_dir"] VALID_TYPES = ["file", "git_dir"]
TIME_NOW = datetime.now() TIME_NOW = datetime.now()
def request_wrapper(url):
for i in range(1,4): def request_wrapper(url):
for i in range(1, 4):
r = requests.get(url) r = requests.get(url)
if r.status_code == 200: if r.status_code == 200:
# print("[+] Got %s successfully!"%(url)) # print("[+] Got %s successfully!"%(url))
break break
if i == 3: if i == 3:
print("[!] Failed to get %s."%(url)) print("[!] Failed to get %s." % (url))
exit(2) exit(2)
print("[!] Getting %s failed(%i/3)"%(url,i)) print("[!] Getting %s failed(%i/3)" % (url, i))
return r.text return r.text
# Check if the files exists # Check if the files exists
if not os.path.isfile(SOURCE_PATH): if not os.path.isfile(SOURCE_PATH):
print("[!] Sources.json is missing!") print("[!] Sources.json is missing!")
@ -49,20 +50,20 @@ for source in SOURCES:
task_name = source["name"] task_name = source["name"]
source_keys = source.keys() source_keys = source.keys()
if not task_name in STATUS.keys(): if task_name not in STATUS.keys():
print(f"[+] Queuing task {task_name} as task was never checked before") print(f"[+] Queuing task {task_name} as task was never checked before")
to_check.append(source) to_check.append(source)
continue continue
if not "output" in source_keys or not isinstance(source["output"], str): if "output" not in source_keys or not isinstance(source["output"], str):
print(f"[!] Skipping task {task_name} as output field is missing/invalid") print(f"[!] Skipping task {task_name} as output field is missing/invalid")
continue continue
if not "type" in source_keys or not isinstance(source["type"], str): if "type" not in source_keys or not isinstance(source["type"], str):
print(f"[!] Skipping task {task_name} as type field is missing/invalid") print(f"[!] Skipping task {task_name} as type field is missing/invalid")
continue continue
if not source["type"] in VALID_TYPES: if source["type"] not in VALID_TYPES:
print(f"[!] Skipping task {task_name} as type is invalid") print(f"[!] Skipping task {task_name} as type is invalid")
continue continue
@ -71,45 +72,62 @@ for source in SOURCES:
continue continue
if source["type"].startswith("git_") and not source["source"].endswith(".git"): if source["type"].startswith("git_") and not source["source"].endswith(".git"):
print(f"[!] Skipping task {task_name} as a git task was defined with a non git url.") print(
f"[!] Skipping task {task_name} as a git task was defined with a non git url."
)
continue continue
if not "last_update" in STATUS[task_name].keys() or not isinstance(STATUS[task_name]["last_update"], int): if "last_update" not in STATUS[task_name].keys() or not isinstance(
STATUS[task_name]["last_update"], int
):
print(f"[!] Queuing task {task_name} as last_update field is missing/invalid") print(f"[!] Queuing task {task_name} as last_update field is missing/invalid")
to_check.append(source) to_check.append(source)
continue continue
if not ("frequency" in source_keys) ^ ("update_time" in source_keys): if not ("frequency" in source_keys) ^ ("update_time" in source_keys):
print(f"[!] Skipping task {task_name} as only frequency or update_time can be specified") print(
f"[!] Skipping task {task_name} as only frequency or update_time can be specified"
)
continue continue
if "frequency" in source_keys and isinstance(source["frequency"], str): if "frequency" in source_keys and isinstance(source["frequency"], str):
regex_match = re.search(FREQUENCY_REGEX, source["frequency"]) regex_match = re.search(FREQUENCY_REGEX, source["frequency"])
if not regex_match: if not regex_match:
print(f"[!] Skipping task {task_name} as frequency field contains invalid formatting of days and hours") print(
f"[!] Skipping task {task_name} as frequency field contains invalid formatting of days and hours"
)
continue continue
days, _, hours, _ = regex_match.groups() days, _, hours, _ = regex_match.groups()
days = bool(days) | 0 days = int(days or 0)
hours = bool(hours) | 0 hours = int(hours or 0)
next_update_time = datetime.fromtimestamp(STATUS[task_name]["last_update"]) + timedelta(days=days, hours=hours) next_update_time = datetime.fromtimestamp(
time_from_update = TIME_NOW - next_update_time STATUS[task_name]["last_update"]
time_to_update = next_update_time - TIME_NOW ) + timedelta(days=days, hours=hours)
if TIME_NOW < next_update_time: time_to_update = int((next_update_time - TIME_NOW).total_seconds())
if time_to_update.seconds <= 300:
print(f"[+] Queuing task {task_name} as it is less than 5 minutes to update. ({time_to_update.seconds} seconds to update)") if TIME_NOW > next_update_time:
print(
f"[+] Queuing task {task_name} as it is {time_to_update} seconds after scheduled update time."
)
to_check.append(source) to_check.append(source)
continue continue
print(f"[!] Skipping task {task_name} as it is more than 5 minutes to update ({time_to_update.seconds} seconds to update)") elif time_to_update <= 300:
print(
f"[+] Queuing task {task_name} as it is less than 5 minutes to update. ({time_to_update} seconds to update)"
)
to_check.append(source)
continue continue
print(f"[+] Queuing task {task_name} as it is {time_to_update.seconds} seconds after scheduled update time.") print(
to_check.append(source) f"[!] Skipping task {task_name} as it is more than 5 minutes to update ({time_to_update} seconds to update)"
)
continue
elif "update_time" in source_keys and isinstance(source["update_time"], str): elif "update_time" in source_keys and isinstance(source["update_time"], str):
update_time = source["update_time"] update_time = source["update_time"]
@ -121,17 +139,22 @@ for source in SOURCES:
hours = int(update_time[:2]) hours = int(update_time[:2])
minutes = int(update_time[2:]) minutes = int(update_time[2:])
if not hours in range(1, 25): if hours not in range(1, 25):
print(f"[!] Skipping task {task_name} as hours is not in range 1-24.") print(f"[!] Skipping task {task_name} as hours is not in range 1-24.")
continue continue
if not minutes in range(1, 61): if minutes not in range(1, 61):
print(f"[!] Skipping task {task_name} as minutes is not in range 1-60.") print(f"[!] Skipping task {task_name} as minutes is not in range 1-60.")
continue continue
scheduled_update_time = TIME_NOW.replace(hour=hours, minute=minutes) scheduled_update_time = TIME_NOW.replace(hour=hours, minute=minutes)
if TIME_NOW <= scheduled_update_time and TIME_NOW + timedelta(hours=1) >= scheduled_update_time: if (
print(f"[+] Queuing task {task_name} as update time is within the next hour") TIME_NOW <= scheduled_update_time
and TIME_NOW + timedelta(hours=1) >= scheduled_update_time
):
print(
f"[+] Queuing task {task_name} as update time is within the next hour"
)
to_check.append(source) to_check.append(source)
continue continue
@ -140,7 +163,7 @@ for source in SOURCES:
continue continue
if len(to_check) == 0: if len(to_check) == 0:
print(f"[!] No task were queued. Exiting.") print("[!] No task were queued. Exiting.")
exit() exit()
print(f"[+] Queued a total of {len(to_check)} tasks to run.") print(f"[+] Queued a total of {len(to_check)} tasks to run.")
@ -148,7 +171,7 @@ print(f"[+] Queued a total of {len(to_check)} tasks to run.")
for task in to_check: for task in to_check:
print(f"[+] Starting task {task['name']}") print(f"[+] Starting task {task['name']}")
if not task["name"] in STATUS.keys(): if task["name"] not in STATUS.keys():
STATUS[task["name"]] = {} STATUS[task["name"]] = {}
task_type = task["type"] task_type = task["type"]
@ -156,16 +179,18 @@ for task in to_check:
if task_type == "file": if task_type == "file":
content = request_wrapper(task["source"]) content = request_wrapper(task["source"])
open(task["output"], "w").write(content) open(task["output"], "w").write(content)
print(f"[+] Saved file to output location") print("[+] Saved file to output location")
STATUS[task["name"]]["last_update"] = int(datetime.now().timestamp()) STATUS[task["name"]]["last_update"] = int(datetime.now().timestamp())
elif task_type == "git_dir": elif task_type == "git_dir":
if not os.path.exists(task['output']): if not os.path.exists(task["output"]):
print(f"[+] Making directory {task['output']}") print(f"[+] Making directory {task['output']}")
os.makedirs(task["output"]) os.makedirs(task["output"])
subprocess.run(["git", "clone", "-q", "--depth=1", task["source"]], cwd=task["output"]) subprocess.run(
["git", "clone", "-q", "--depth=1", task["source"]], cwd=task["output"]
)
STATUS[task["name"]]["last_update"] = int(datetime.now().timestamp()) STATUS[task["name"]]["last_update"] = int(datetime.now().timestamp())
if task["post_run_script"]: if task["post_run_script"]: