【攻防技术】信息收集之公众号自动收集

# https://open.tianyancha.com/open/834def GetWechatOfficialAccount(self, keyword, pageNum=1, pageSize=20):    url = "http://open.api.tianyancha.com/services/open/ipr/publicWeChat/2.0?keyword=%s&pageNum=%s&pageSize=%s" % (keyword, pageNum, pageSize)    headers={'Authorization': {天眼查Token}}    try:        response = requests.get(url, headers=headers)    except Exception as err:        logger.error(str(err))        return None    data = response.json()

import urllib.parseimport requestsfrom lxml import etreefrom fake_useragent import UserAgentimport time,sys,os,math,json,random
#生成随机UAua = UserAgent(browsers=["chrome", "edge", "internet explorer", "firefox", "safari"])global_headers = {    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",    "Accept-Encoding": "gzip, deflate, br",    "Accept-Language": "en-US,en;q=0.9",    "Cache-Control": "max-age=0",    "Connection": "keep-alive",    "Cookie": "SUID=10800C75492CA20A000000006145E582; SUV=1641002706609755; ssuid=8628497906; IPLOC=CN1200; cuid=AAERmghDQwAAAAqgHl6xlgAASQU=; ABTEST=0|1684314665|v1; JSESSIONID=aaaXmp_IROcwx7NZRDDEy; PHPSESSID=snjidnapfi0p1lsac6jgb8huu1; SNUID=8A0A5AACD7DD2C295C858FD2D8397552; ariaDefaultTheme=undefined",    "Referer": "https://weixin.sogou.com",    "Sec-Fetch-Dest": "document",    "Sec-Fetch-Mode": "navigate",    "Sec-Fetch-Site": "none",    "Sec-Fetch-User": "?1",    "Upgrade-Insecure-Requests": "1",    "User-Agent": ua.random,    "sec-ch-ua": '"Google Chrome";v="113", "Chromium";v="113", "Not-A.Brand";v="24"',    "sec-ch-ua-mobile": "?0",    "sec-ch-ua-platform": '"Windows"',    "X-Forwarded-For": "110.242.68.66"}
#https://weixin.sogou.com/weixin?query=%E6%B8%A4%E6%B5%B7%E9%93%B6%E8%A1%8C&_sug_type_=&s_from=input&_sug_=n&type=1&ie=utf8&page=2#将中文转为url编码def Chinese2Url(chinese_context):    return urllib.parse.quote(chinese_context)
#获取页面的公众号名称#页面编号为1，返回获取的数量和第1页公众号列表#页面编号大于1，返回当页的公众号列表def GetWechatofficialAccount(keyword_url, headers=global_headers, page=1):    url = "https://weixin.sogou.com/weixin?query="+keyword_url+"&_sug_type_=&s_from=input&_sug_=n&type=1&ie=utf8&page=" + str(page)    xpath_get_official_account = "//li/div/div[2]/p[1]/a/text()"    rep = requests.get(url, headers=headers)    if "请输入验证码" in rep.text:        headers = GetNewCookieHeaders()        rep = requests.get(url, headers=headers)        #更新全局变量，headers和代理        global global_headers        global_headers = headers        req = requests.get(url, headers=headers)    response = rep.text.replace("<em><!--red_beg-->","").replace("<!--red_end--></em>","")    etree_obj = etree.HTML(response)    wechat_official_account = etree_obj.xpath(xpath_get_official_account)    if('<img src="/new/pc/images/bg_404_2.png"' in response):        print("No result")        sys.exit()    if(page == 1):        xpath_get_page_num = "//div[@class='mun']/text()"        item_num = etree_obj.xpath(xpath_get_page_num)[0].replace("找到","").replace("约","").replace("条结果","")        print("item number is %s" % item_num)        return int(item_num),wechat_official_account    else:        return wechat_official_account
def GetNewCookieHeaders():    headers = global_headers    cookie_split = headers["Cookie"].split("; ")    for item in cookie_split:        if "SNUID" in item:            cookie_split.remove(item)            cookie_split.append("SNUID=%s" % GetSougoCookie()["SNUID"])            break    headers["Cookie"] = "; ".join(cookie_split)    return headers
def GetSougoCookie():    # 搜狗视频url，获取新的SNUID    url = "https://v.sogou.com/?forceredirect=2&ie=utf8"    rst = requests.get(url=url,                       headers=global_headers,                       allow_redirects=False)    cookies = rst.cookies.get_dict()    print("Get new cookie SNUID %s" % cookies["SNUID"])    return {"SNUID":cookies["SNUID"]}
if __name__ == '__main__':    keyword_url = Chinese2Url("中国移动")    item_num,wechat_official_account = GetWechatofficialAccount(keyword_url,headers=global_headers,page=1)    page_num = 10    if item_num < 100:        page_num = math.ceil(item_num/10)    if page_num > 1:        page = 2        while (page <= page_num):            wechat_official_account = sum([wechat_official_account,GetWechatofficialAccount(keyword_url,headers=global_headers,page=page)],[])            print("page = %s" % page)            page += 1    wechat_official_account = list(set(wechat_official_account))    print(wechat_official_account)