mirror of
https://git.freebsd.org/ports.git
synced 2025-06-16 02:00:30 -04:00
9ff6a0ecc0
Despite the name, the crawler gets info for more services than just DNS:
- DNS:
- all A/AAAA records (for the 2nd level domain and www.subdomain), annotated
with GeoIP
- TXT records (with SPF and DMARC parsed for easier filtering)
- TLSA (for the 2nd level domain and www.subdomain)
- MX
- DNSSEC validation
- nameservers:
- each server IP annotated with GeoIP
- HOSTNAME.BIND, VERSION.BIND, AUTHORS.BIND and fortune (also for all IPs)
- users can add custom additional RRs in the config file
- E-mail (for every server from MX):
- SMTP server banners (optional, ports are configurable)
- TLSA records
- Web:
- HTTP status & headers (inc. parsed cookies) for ports 80 & 443 on each IP
from A/AAAA records
- certificate info for HTTPS (optionally with an entire cert chain)
- webpage content (optional)
- everything of the above is saved for each step in the redirect history --
the crawler follows redirects until it gets a non-redirecting status or hits
a configurable limit
WWW: https://gitlab.nic.cz/adam/dns-crawler
41 lines
1.1 KiB
Python
41 lines
1.1 KiB
Python
--- setup.py.orig 2020-07-10 06:16:32 UTC
|
|
+++ setup.py
|
|
@@ -39,23 +39,23 @@ setup(
|
|
]
|
|
},
|
|
install_requires=[
|
|
- "asn1crypto==1.3.0",
|
|
- "cert_human==1.0.7",
|
|
- "cryptography==2.8",
|
|
+ "asn1crypto>=1.3.0",
|
|
+ "cert_human>=1.0.7",
|
|
+ "cryptography>=2.8",
|
|
"dnspython",
|
|
- "ecdsa==0.15",
|
|
- "forcediphttpsadapter==1.0.1",
|
|
- "geoip2==3.0.0",
|
|
+ "ecdsa>=0.15",
|
|
+ "forcediphttpsadapter>=1.0.1",
|
|
+ "geoip2>=3.0.0",
|
|
"hstspreload",
|
|
- "idna==2.9",
|
|
- "pyaml==19.12.0",
|
|
- "PyICU==2.4.3",
|
|
- "pycryptodome==3.9.7",
|
|
- "pyopenssl==19.1.0",
|
|
- "redis==3.4.1",
|
|
- "requests_toolbelt==0.9.1",
|
|
- "requests==2.23.0",
|
|
- "rq==1.2.2",
|
|
+ "idna>=2.9",
|
|
+ "pyaml>=19.12.0",
|
|
+ "PyICU>=2.4.3",
|
|
+ "pycryptodome>=3.9.7",
|
|
+ "pyopenssl>=19.1.0",
|
|
+ "redis>=3.4.1",
|
|
+ "requests_toolbelt>=0.9.1",
|
|
+ "requests>=2.23.0",
|
|
+ "rq>=1.2.2",
|
|
],
|
|
keywords=["crawler", "dns", "http", "https"],
|
|
classifiers=[
|