从代码角度看各类子域名收集工具

supported_engines = {'baidu': BaiduEnum,
'yahoo': YahooEnum,
'google': GoogleEnum,
'bing': BingEnum,
'ask': AskEnum,
'netcraft': NetcraftEnum,
'dnsdumpster': DNSdumpster,
'virustotal': Virustotal,
'threatcrowd': ThreatCrowd,
'ssl': CrtSearch,
'passivedns': PassiveDNS
}

#Using a 32 char string every time may be too predictable.
x = uuid.uuid4().hex[0:random.randint(6, 32)]
testdomain = "%s.%s" % (x, host)

class DNSTransfer(object):
def __init__(self, domain):
self.domain = domain

def transfer_info(self):
ret_zones = list()
try:
nss = dns.resolver.query(self.domain, 'NS')
nameservers = [str(ns) for ns in nss]
ns_addr = dns.resolver.query(nameservers[0], 'A')
# dnspython 的 bug，需要设置 lifetime 参数
zones = dns.zone.from_xfr(dns.query.xfr(ns_addr, self.domain, relativize=False, timeout=2, lifetime=2), check_origin=False)
names = zones.nodes.keys()
for n in names:
subdomain = ''
for t in range(0, len(n) - 1):
if subdomain != '':
subdomain += '.'
subdomain += str(n[t].decode())
if subdomain != self.domain:
ret_zones.append(subdomain)
return ret_zones
except BaseException:
return []

class CAInfo(object):
def __init__(self, domain):
self.domain = domain

def dns_resolve(self):
padding_domain = 'www.' + self.domain
# loop = asyncio.get_event_loop()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
resolver = aiodns.DNSResolver(loop=loop)
f = resolver.query(padding_domain, 'A')
result = loop.run_until_complete(f)
return result[0].host

def get_cert_info_by_ip(self, ip):
s = socket.socket()
s.settimeout(2)
base_dir = os.path.dirname(os.path.abspath(__file__))
cert_path = base_dir + '/cacert.pem'
connect = ssl.wrap_socket(s, cert_reqs=ssl.CERT_REQUIRED, ca_certs=cert_path)
connect.settimeout(2)
connect.connect((ip, 443))
cert_data = connect.getpeercert().get('subjectAltName')
return cert_data

def get_ca_domain_info(self):
domain_list = list()
try:
ip = self.dns_resolve()
cert_data = self.get_cert_info_by_ip(ip)
except Exception as e:
return domain_list

for domain_info in cert_data:
hostname = domain_info[1]
if not hostname.startswith('*') and hostname.endswith(self.domain):
domain_list.append(hostname)

return domain_list

def get_subdomains(self):
subs = list()
subdomain_list = self.get_ca_domain_info()
for sub in subdomain_list:
subs.append(sub[:len(sub) - len(self.domain) - 1])
return subs

def check(self, dns):
logger.info("Checking if DNS server {dns} is available".format(dns=dns))
msg = b'x5cx6dx01x00x00x01x00x00x00x00x00x00x03wwwx05baidux03comx00x00x01x00x01'
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.settimeout(3)
repeat = {
1: 'first',
2: 'second',
3: 'third'
}
for i in range(3):
logger.info("Sending message to DNS server a {times} time".format(times=repeat[i + 1]))
sock.sendto(msg, (dns, 53))
try:
sock.recv(4096)
break
except socket.timeout as e:
logger.warning('Failed!')
if i == 2:
return False
return True

from difflib import SequenceMatcher
# RSC ratio
self.rsc_ratio = 0.8
self.wildcard_html # 获取一个随机子域名的html
ratio = SequenceMatcher(None, html, self.wildcard_html).real_quick_ratio()
ratio = round(ratio, 3)
if ratio > self.rsc_ratio:
# passed
logger.debug('{r} RSC ratio: {ratio} (passed) {sub}'.format(r=self.remainder, sub=sub_domain, ratio=ratio))
else:
# added

def match_subdomains(domain, html, distinct=True, fuzzy=True):
"""
    Use regexp to match subdomains

    :param  str domain: main domain
    :param  str html: response html text
    :param  bool distinct: deduplicate results or not (default True)
    :param  bool fuzzy: fuzzy match subdomain or not (default True)
    :return set/list: result set or list
    """
logger.log('TRACE', f'Use regexp to match subdomains in the response body')
if fuzzy:
regexp = r'(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?.){0,}' 
+ domain.replace('.', r'.')
result = re.findall(regexp, html, re.I)
if not result:
return set()
deal = map(lambda s: s.lower(), result)
if distinct:
return set(deal)
else:
return list(deal)
else:
regexp = r'(?:>|"|'|=|,)(?:http://|https://)?' 
r'(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?.){0,}' 
+ domain.replace('.', r'.')
result = re.findall(regexp, html, re.I)
if not result:
return set()
regexp = r'(?:http://|https://)'
deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
if distinct:
return set(deal)
else:
return list(deal)

wildcard_ips = list()  # 泛解析IP列表
wildcard_ttl = int()  # 泛解析TTL整型值
ns_list = query_domain_ns(self.domain) # 查询域名NS记录
ns_ip_list = query_domain_ns_a(ns_list)  # DNS权威名称服务器对应A记录列表
self.enable_wildcard = detect_wildcard(domain, ns_ip_list) # 通过域名指定NS查询是否有泛解析

if self.enable_wildcard:
wildcard_ips, wildcard_ttl = collect_wildcard_record(domain,
ns_ip_list)
# 收集泛解析范围，当大部分泛解析记录(80%)达到同一IP出现两次以上，则返回该IP以及TTL
ns_path = get_nameservers_path(self.enable_wildcard, ns_ip_list)

# 生成字典
dict_set = self.gen_brute_dict(domain)
dict_len = len(dict_set)
dict_name = f'generated_subdomains_{domain}_{timestring}.txt'
dict_path = temp_dir.joinpath(dict_name)
save_brute_dict(dict_path, dict_set)
del dict_set

# 调用massdns进行扫描
output_name = f'resolved_result_{domain}_{timestring}.json'
output_path = temp_dir.joinpath(output_name)
log_path = result_dir.joinpath('massdns.log')
check_dict()
logger.log('INFOR', f'Running massdns to brute subdomains')
utils.call_massdns(massdns_path, dict_path, ns_path, output_path,
log_path, quiet_mode=self.quite,
process_num=self.process_num,
concurrent_num=self.concurrent_num)

[
{
"name":"github", 
"cname":["github.io", "github.map.fastly.net"], 
"response":["There isn't a GitHub Pages site here.", "For root URLs (like http://example.com/) you must provide an index.html file"]
},
{
"name":"heroku", 
"cname":["herokudns.com", "herokussl.com", "herokuapp.com"], 
"response":["There's nothing here, yet.", "herokucdn.com/error-pages/no-such-app.html", "<title>No such app</title>"]
},
{
"name":"unbounce",
"cname":["unbouncepages.com"],
"response":["Sorry, the page you were looking for doesn’t exist.", "The requested URL was not found on this server"]
},
{
"name":"tumblr",
"cname":["tumblr.com"],
"response":["There's nothing here.", "Whatever you were looking for doesn't currently exist at this address."]
},
{
"name":"shopify",
"cname":["myshopify.com"],
"response":["Sorry, this shop is currently unavailable.", "Only one step left!"]
},
{
"name":"instapage",
"cname":["pageserve.co", "secure.pageserve.co", "https://instapage.com/"],
"response":["Looks Like You're Lost","The page you're looking for is no longer available."]
},
{
"name":"desk",
"cname":["desk.com"],
"response":["Please try again or try Desk.com free for 14 days.", "Sorry, We Couldn't Find That Page"]
},
{
"name":"campaignmonitor",
"cname":["createsend.com", "name.createsend.com"],
"response":["Double check the URL", "<strong>Trying to access your account?</strong>"]
},
{
"name":"cargocollective",
"cname":["cargocollective.com"],
"response":["404 Not Found"]
},
{
"name":"statuspage",
"cname":["statuspage.io"],
"response":["Better Status Communication", "You are being <a href="https://www.statuspage.io">redirected"]
},
{
"name":"amazonaws",
"cname":["amazonaws.com"],
"response":["NoSuchBucket", "The specified bucket does not exist"]
},
{
"name":"bitbucket",
"cname":["bitbucket.org"],  
"response":["The page you have requested does not exist","Repository not found"]
},
{
"name":"smartling",
"cname":["smartling.com"],
"response":["Domain is not configured"]
},
{
"name":"acquia",
"cname":["acquia.com"],
"response":["If you are an Acquia Cloud customer and expect to see your site at this address","The site you are looking for could not be found."]
},
{
"name":"fastly",
"cname":["fastly.net"],
"response":["Please check that this domain has been added to a service", "Fastly error: unknown domain"]
},
{
"name":"pantheon",
"cname":["pantheonsite.io"],
"response":["The gods are wise", "The gods are wise, but do not know of the site which you seek."]
},
{
"name":"zendesk",
"cname":["zendesk.com"],
"response":["Help Center Closed"]
},
{
"name":"uservoice",
"cname":["uservoice.com"],
"response":["This UserVoice subdomain is currently available!"]
},
{
"name":"ghost",
"cname":["ghost.io"],
"response":["The thing you were looking for is no longer here", "The thing you were looking for is no longer here, or never was"]
},
{
"name":"pingdom",
"cname":["stats.pingdom.com"],
"response":["pingdom"]
},
{
"name":"tilda",
"cname":["tilda.ws"],
"response":["Domain has been assigned"]
},
{
"name":"wordpress",
"cname":["wordpress.com"],  
"response":["Do you want to register"]
},
{
"name":"teamwork",
"cname":["teamwork.com"],
"response":["Oops - We didn't find your site."]
},
{
"name":"helpjuice",
"cname":["helpjuice.com"],
"response":["We could not find what you're looking for."]
},
{
"name":"helpscout",
"cname":["helpscoutdocs.com"],
"response":["No settings were found for this company:"]
},
{
"name":"cargo",
"cname":["cargocollective.com"],
"response":["If you're moving your domain away from Cargo you must make this configuration through your registrar's DNS control panel."]
},
{
"name":"feedpress",
"cname":["redirect.feedpress.me"],
"response":["The feed has not been found."]
},
{
"name":"surge",
"cname":["surge.sh"],
"response":["project not found"]
},
{
"name":"surveygizmo",
"cname":["privatedomain.sgizmo.com", "privatedomain.surveygizmo.eu", "privatedomain.sgizmoca.com"],
"response":["data-html-name"]
},
{
"name":"mashery",
"cname":["mashery.com"],
"response":["Unrecognized domain <strong>"]
},
{
"name":"intercom",
"cname":["custom.intercom.help"],
"response":["This page is reserved for artistic dogs.","<h1 class="headline">Uh oh. That page doesn’t exist.</h1>"]
},
{
"name":"webflow",
"cname":["proxy.webflow.io"],
"response":["<p class="description">The page you are looking for doesn't exist or has been moved.</p>"]
},
{
"name":"kajabi",
"cname":["endpoint.mykajabi.com"],
"response":["<h1>The page you were looking for doesn't exist.</h1>"]
},
{
"name":"thinkific",
"cname":["thinkific.com"],
"response":["You may have mistyped the address or the page may have moved."]
},
{
"name":"tave",
"cname":["clientaccess.tave.com"],
"response":["<h1>Error 404: Page Not Found</h1>"]
},
{
"name":"wishpond",
"cname":["wishpond.com"],
"response":["https://www.wishpond.com/404?campaign=true"]
},
{
"name":"aftership",
"cname":["aftership.com"],
"response":["Oops.</h2><p class="text-muted text-tight">The page you're looking for doesn't exist."]
},
{
"name":"aha",
"cname":["ideas.aha.io"],
"response":["There is no portal here ... sending you back to Aha!"]
},
{
"name":"brightcove",
"cname":["brightcovegallery.com", "gallery.video", "bcvp0rtal.com"],
"response":["<p class="bc-gallery-error-code">Error Code: 404</p>"]
},
{
"name":"bigcartel",
"cname":["bigcartel.com"],
"response":["<h1>Oops! We couldn&#8217;t find that page.</h1>"]
},
{
"name":"activecompaign",
"cname":["activehosted.com"],
"response":["alt="LIGHTTPD - fly light.""]
},
{
"name":"compaignmonitor",
"cname":["createsend.com"],
"response":["Double check the URL or <a href="mailto:[email protected]"]
},
{
"name":"simplebooklet",
"cname":["simplebooklet.com"],
"response":["We can't find this <a href="https://simplebooklet.com"]
},
{
"name":"getresponse",
"cname":[".gr8.com"],
"response":["With GetResponse Landing Pages, lead generation has never been easier"]
},
{
"name":"vend",
"cname":["vendecommerce.com"],
"response":["Looks like you've traveled too far into cyberspace."]
},
{
"name":"jetbrains",
"cname":["myjetbrains.com"],
"response":["is not a registered InCloud YouTrack.","is not a registered InCloud YouTrack."]
},
{
"name":"azure",
"cname":["azurewebsites.net",
".cloudapp.net",
".cloudapp.azure.com",
".trafficmanager.net",
".blob.core.windows.net",
".azure-api.net",
".azurehdinsight.net",
".azureedge.net"],
"response":["404 Web Site not found"]
},
{
"name":"readme",
"cname":["readme.io"],
"response":["Project doesnt exist... yet!"]
}
]

{
"name": "wordpress",
"author": "fate0",
"version": "0.1.0",
"description": "wordpress 是世界上最为广泛使用的博客系统",
"website": "http://www.wordpress.org/",
"matches": [],
"condition": "0 and (1 and not 2)",
"implies": "PHP",
"excludes": "Apache"
}

pip install aiofiles

import asyncio
import aiofiles


async def scan(line):
print(line)
await asyncio.sleep(3) # 模拟耗时


async def main():
path = "subnames.txt"
async with aiofiles.open(path, 'r') as f:
async for line in f:
await scan(line.strip())


if __name__ == '__main__':
asyncio.get_event_loop().run_until_complete(main())

var DefaultSources = []string{
"alienvault",
"archiveis",
"binaryedge",
"bufferover",
"censys",
"certspotter",
"certspotterold",
"commoncrawl",
"crtsh",
"digicert",
"dnsdumpster",
"dnsdb",
"entrust",
"hackertarget",
"ipv4info",
"intelx",
"passivetotal",
"rapiddns",
"securitytrails",
"shodan",
"sitedossier",
"spyse",
"sublist3r",
"threatcrowd",
"threatminer",
"urlscan",
"virustotal",
"waybackarchive",
"zoomeye",
}

type Agent struct {
sources map[string]subscraping.Source
}

subfinder -d http://hackerone.com -silent | dnsprobe -f domain.txt

subfinder -d http://hackerone.com -silent | dnsprobe -f domain.txt