为被动扫描器量身打造一款爬虫 —LSpider

self.chrome_options.add_argument('--headless')self.chrome_options.add_argument('--disable-gpu')self.chrome_options.add_argument('--no-sandbox')self.chrome_options.add_argument('--disable-images')self.chrome_options.add_argument('--ignore-certificate-errors')self.chrome_options.add_argument('--allow-running-insecure-content')self.chrome_options.add_argument('blink-settings=imagesEnabled=false')self.chrome_options.add_argument('--omnibox-popup-count="5"')self.chrome_options.add_argument("--disable-popup-blocking")self.chrome_options.add_argument("--disable-web-security")self.chrome_options.add_argument("--disk-cache-size=1000")

if os.name == 'nt':    chrome_downloadfile_path = "./tmp"else:    chrome_downloadfile_path = '/dev/null'
prefs = {    'download.prompt_for_download': True,    'profile.default_content_settings.popups': 0,    'download.default_directory': chrome_downloadfile_path}

desired_capabilities = self.chrome_options.to_capabilities()if IS_OPEN_CHROME_PROXY:    logger.info("[Chrome Headless] Proxy {} init".format(CHROME_PROXY))
    desired_capabilities['acceptSslCerts'] = True    desired_capabilities['acceptInsecureCerts'] = True    desired_capabilities['proxy'] = {        "httpProxy": CHROME_PROXY,        "ftpProxy": CHROME_PROXY,        "sslProxy": CHROME_PROXY,        "noProxy": None,        "proxyType": "MANUAL",        "class": "org.openqa.selenium.Proxy",        "autodetect": False,    }

self.driver.set_page_load_timeout(15)self.driver.set_script_timeout(5)

self.origin_url = urlself.driver.implicitly_wait(5)self.driver.get(url)
if cookies:    self.add_cookie(cookies)    self.driver.implicitly_wait(10)    self.driver.get(url)

links = self.driver.find_elements_by_xpath('//a')link = links[i]
href = link.get_attribute('href')self.driver.execute_script(    "atags = document.getElementsByTagName('a');for(i=0;i<=atags.length;i++) { if(atags[i]){atags[i].setAttribute('target', '')}}")
if link.is_displayed() and link.is_enabled():    link.click()
    self.check_back()

if submit_button.is_displayed() and submit_button.is_enabled():    action = ActionChains(self.driver)    action.move_to_element(submit_button).perform()
    submit_button.click()

form    ->  文本中出现login、登录、sign等button  ->  outerHTML出现login、user、pass等input   ->  outerHTML出现登录、注册等关键词a       ->  文本出现login、登录、sign等

inputs = self.driver.find_elements_by_xpath("//input")self.driver.execute_script(    "itags = document.getElementsByTagName('input');for(i=0;i<=itags.length;i++) { if(itags[i]){itags[i].removeAttribute('style')}}")
input_lens = len(inputs)
if not inputs:    return
for i in range(input_lens):    try:        input = inputs[i]
        # 移动鼠标        # 如果标签没有隐藏，那么移动鼠标        if input.is_enabled() and input.is_displayed():
            action = ActionChains(self.driver)            action.move_to_element(input).perform()

BLACK_DOMAIN_NAME_LIST = ['docs', 'image', 'static', 'blogs']

https://lorexxar.cn/2020/10/30/whitebox-2/
就会被泛化为AAAB

BLACK_PATH_NAME = ['static', 'upload', 'docs', 'js', 'css', 'font', 'image']

https://lorexxar.cn/2020/10/30/whitebox-2/a.php