信息收集之路-B段信息及历史域名

#!/usr/bin/env python3# -*- coding: utf-8 -*-# @File : Stand.py# @Author : Null# @Time : 2021/4/15 13:43# @Software: PyChar
import requestsimport reimport sysimport timefrom lxml import etreefrom bs4 import BeautifulSoup

def scan(url):  #C段历史域名获取    try:        headers = {            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'}        a = re.findall('(d{0,3}).(d{0,3}).(d{0,3}).(d{0,3})', url)        urls = a[0][0] + '.' + a[0][1] + '.' + a[0][2] + '.' + '0' + '/24' #由于是爬取网站查询结果只能照着格式来，所以还是重构了ip        vuln_url = "https://chapangzhan.com/"+str(urls)             #构造请求链接        conn = requests.session()                               #请求会话        r = conn.get(url=vuln_url, headers=headers)        soup = BeautifulSoup(r.text, 'lxml')                    #使用BrautifulSoup来筛选并获取内容        h1 = soup.h1.text        p = soup.p.text        print(h1+p)                                             #输出所获得的内容 例:  xx.xx.xx.* 网站大全中国**长沙电信        for itme in soup.find_all("a"):            itmes = str(itme.get("href"))            res = re.search(r"https://ipchaxun.com/",itmes)     #以下操作差不多 只是请求的链接不一样            if None != res:                r2 = conn.get(url=itmes, headers=headers)                soup2 = BeautifulSoup(r2.text,'lxml')                for itme2 in soup2.find_all("p"):                    itmes2 = itme2.get_text()                    res2 = re.search(r"(d{4}-d{1,2}-d{1,2})",itmes2)  #在获取内容的时候 可能会有很多重复的 所以得找关键字去进行一定的匹配                    if None != res2:                        print('C段ip：n' + itme.get_text())                        print("相关域名:")                        print(itme2.a.text + 'n')                        with open('历史域名表.txt', 'a') as c:                            c.write(itme2.a.text + 'n')                        with open('历史域名详细表.txt', 'a') as b:
                            b.write(itme.get_text() + '  ---  ' + itme2.a.text + 'n')
    except Exception as e:        pass

def csan_ip(url): #B段归属获取    headers = {        'User-Agent': 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)'}
    url_z = "http://ip.tool.chinaz.com/" #利用站长之家的功能    a = re.findall('(d{0,3}).(d{0,3}).(d{0,3}).(d{0,3})', url)   #匹配每个ip的每个段的值    j = 0                                                               #循环值初始化    ip_new0 = a[0][0] + '.' + a[0][1] + '.'                             #重构ip的B段值    ip_new1 = '.' + str(1)    while j <= 254:                                                     #通过循环来对重构后的ip        ip_news = ip_new0 + str(j) + ip_new1        url_zhen = url_z + ip_news        try:            response = requests.get(url_zhen, headers=headers, timeout=5).text      #发起请求            html = etree.HTML(response)                                             #将其转化为html            a = html.xpath('//*[@id="leftinfo"]/div[3]/div[2]/div[2]/span[4]/p/text()')  #并通过xpath去获取我们获取我们说需要的信息
            #if "xx市" in a[0]:                          # "xx市"  为判断条件 自行更改 筛选            with open(url + 'B段.txt', 'a') as c:        #打开文件并写入查询结果   注：作者很懒所以每次使用脚本都会记录在同一文件底下无限累加                c.write(a[0] + ' ==> ' + ip_news + 'n')            scan(ip_news)                               #C段历史域名获取                #print(a[0] + ' --  ' + ip_news)            print('==================================================')        except:            pass        j = j + 1        time.sleep(0.5)  #延时自己调
def main():  #程序入口    try:        add = sys.argv[1]        if add == '-i':            add_url = sys.argv[2]            csan_ip(add_url)  # ip相关信息收集        elif add == '-r':            add_url = sys.argv[2]            with open(add_url,'r') as f:                ff = f.read()                lists = ff.split('n')                for i in lists:                    csan_ip(i)                    print('-=======================================')
        elif add == '-h':            print('''python3 Btaget.py -h                    #帮助python3 Btaget.py -i [ip]              #单个ip扫描python3 Btaget.py -r [绝对路径txt]      #批量ip扫描               ''')    except Exception as e:        print('''python3 Btaget.py -h                    #帮助python3 Btaget.py -i [ip]               #单个ip扫描python3 Btaget.py -r [绝对路径txt]       #批量ip扫描                       ''')

if __name__ == '__main__':    main()

'''参考资料：    https://blog.csdn.net/qq_26736193/article/details/83216518    https://blog.csdn.net/jeikerxiao/article/details/73506238xpath案例：    https://www.cnblogs.com/x-pyue/p/7798819.html'''

from lxml import etree
text = """<li class = "li li-first" ><a href = "link.html">first item</a></li><li class = "li li-first" name = "item" ><a href = "link.html">second item</a></li><li class = "li1 li1-first" name = "item" ><a href = "link.html">second2222 item</a></li><li class = "li1 li1-first" name = "item" ><a href = "link.html">second2222 item</a></li><li class = "li li-first" name = "item" ><a href = "link.html">second item</a></li>"""
html = etree.HTML(text)
result1 = html.xpath('//li[@class="li"]/a/text()')  # 匹配失败
result2 = html.xpath('//li[@class="li li-first"]/a/text()')  # 匹配正确
result3 = html.xpath('//li[contains(@class,"li")]/a/text()')  # 利用contains()函数进行属性多值匹配
result4 = html.xpath('//li[contains(@class,"li") and @name = "item"]/a/text()')  # 多属性匹配


print(html)
print("result1:", result1)
print("result2:", result2)
print("result3:", result3)
print("result4:", result4)

'''参考资料：   https://cuiqingcai.com/1319.html'''from bs4 import BeautifulSoup

bs = BeautifulSoup(r.text, 'lxml')print(bs.prettify()) # 格式化html结构print(bs.title) # 获取title标签的名称print(bs.title.name) # 获取title的nameprint(bs.title.string) # 获取head标签的所有内容print(bs.head)print(bs.div)  # 获取第一个div标签中的所有内容print(bs.div["id"]) # 获取第一个div标签的id的值print(bs.a)print(bs.find_all("a")) # 获取所有的a标签print(bs.find(id="u1")) # 获取id="u1"for item in bs.find_all("a"):    print(item.get("href")) # 获取所有的a标签，并遍历打印a标签中的href的值for item in bs.find_all("a"):    print(item.get_text()) #获取所有的a标签文本内容