CVE漏洞信息与poc链接爬取

def fetch_all_cves(producer, software):    global cve_all    cve_all = []    global herfs_all    herfs_all = []    url = 'https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&isCpeNameSearch=false&cpe_product='           'cpe:/:{}:{}'.format(producer, software)    print(url)    # to get cve number    try:        response = requests.get(url, timeout=60, headers=headers)        if response.status_code == 200:            num = re.findall('"vuln-matching-records-count">(.*)?</strong>', response.text)[0]            msg = 'There are {} cves with {} {}...'.format(num, software, banner)            print(msg)    except:        pass    # fetch all cve no    start_index = index = 0    while start_index < int(num):        url = 'https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&isCpeNameSearch=false&cpe_product='               'cpe:/:{}:{}&startIndex={}'.format(producer, software, start_index)        msg = 'processing page {}/{}...'.format(index + 1, math.ceil(int(num) / 20))        print(msg)        index += 1        start_index = index * 20        try:            response = requests.get(url, timeout=60, headers=headers)            if response.status_code == 200:                cves = re.findall('"vuln-detail-link-d+">(.*)?</a>', response.text)                cve_all.extend(cves)                herfs = 'https://nvd.nist.gov/vuln/detail/' + re.findall('"vuln-detail-link-d+">(.*)?</a>',                                                                         response.text)                herfs_all.extend(herfs)        except:            pass    return url

def get_csrf_json(this_month_id, year, date):  # url = "{}cvrf/{}?api-Version={}".format(base_url, THIS_MONTH_ID, YEAR)  url = "{}cvrf/{}?api-Version={}".format(base_url, this_month_id, year)  headers = {'api-key': api_key, 'Accept': 'application/json'}  response = requests.get(url, headers = headers)  if response.content:    try:      data = json.loads(response.content)      for each_product in data["ProductTree"]["FullProductName"]:        productid = each_product['ProductID']        product_name = each_product['Value']        search_productid_sql = "SELECT * FROM win_product_name WHERE product_id='" + str(productid) + "'"        cur.execute(search_productid_sql)        if cur.rowcount > 0:          pass        else:          insert_product_sql = "INSERT IGNORE INTO win_product_name (product_id,product_name,dt) VALUES('" + str(productid) + "','" + str(product_name) + "','" + TODAY + "')"          cur.execute(insert_product_sql)          db.commit()      for each_cve in data["Vulnerability"]:        cve = each_cve["CVE"]        if re.search('ADV',cve):          pass        else:          cve = each_cve["CVE"]          kblist = []          scorelist = []          product_id_list = str(each_cve["ProductStatuses"][0]["ProductID"]).replace("'","")          product_id_list = product_id_list.replace("[","")          product_id_list = product_id_list.replace("]","")          product_id_list = product_id_list.replace(" ","")          for each_kb in each_cve["Remediations"]:            try:              kb_num = each_kb["Description"]["Value"]              if re.search('Click to Run',kb_num) or re.search('Release Notes',kb_num):                pass              else:                kblist.append('KB{}'.format(kb_num))            except Exception as e:              print("kb",e)          kblist=list(set(kblist))          kblist=str(kblist).replace("'","")          kblist=kblist.replace("[","")          kblist=kblist.replace("]","")          kblist=kblist.replace(" ","")          for each_score in each_cve["CVSSScoreSets"]:            try:              scorelist.append(each_score["BaseScore"])            except Exception as e:              print("score",e)          try:            score_mean = format(sum(scorelist)/len(scorelist),'.1f')          except Exception as e:            print("score_mean",e)            score_mean = 0          # insert_cve_sql = "INSERT INTO win_cve_db (cve,score,product_id_list,kb_list,cvrf_id,dt) VALUES("" + str(cve) + "","" + str(score_mean) + "","" + str(product_id_list) + "","" + str(kblist) + "","" + str(THIS_MONTH_ID) + "","" + TODAY + "")"          # print(insert_cve_sql)          insert_cve_sql = "INSERT IGNORE INTO win_cve_db (cve, score, product_id_list, kb_list, cvrf_id, dt) VALUES (%s, %s, %s, %s, %s, %s)"          cve_data = (cve, score_mean, product_id_list, kblist, THIS_MONTH_ID, TODAY)          cur.execute(insert_cve_sql, cve_data)          # cur.execute(insert_cve_sql)          db.commit()      # db.close()    except json.decoder.JSONDecodeError as e:      print("JSON 解析错误:", e)  else:    print("Response content is empty")

https://github.com/Donvink/Spider.BC