Python学习之路爬虫

"""requests 操作练习"""import requestsimport re#数据的爬取h = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36'}response = requests.get('https://movie.douban.com/chart',headers=h)html_str = response.text#数据解析<a href="https://movie.XX.com/subject/34961898/"  title="汉密尔顿">pattern = re.compile('<a.*?title="(.*?)">')     #  .*? 任意匹配尽可能多的匹配尽可能少的字符result = re.findall(pattern,html_str)print(result)

import requestsimport refrom bs4 import  BeautifulSoupfrom lxml import etree#数据爬取（一些HTTP头的信息）h = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36'}response = requests.get('https://movie.XX.com/chart',headers=h)html_str = response.text#数据解析#正则表达式解析def re_parse(html_str):pattern = re.compile('<a.*?title="(.*?)"')results = re.findall(pattern,html_str)print(results)return results#bs4解析def bs4_parse(html_str):soup = BeautifulSoup(html_str,'lxml')items = soup.find_all(class_='nbg')for item in items:print(item.attrs['title'])#lxml解析def lxml_parse(html_str):html = etree.HTML(html_str)results = html.xpath('//a[@class="nbg"]/@title')print(results)return resultsre_parse(html_str)bs4_parse(html_str)lxml_parse(html_str)

"""whoishttp://whois.chinaz.com/sina.com"""import requestsimport reh = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36'}response = requests.get('http://whois.chinaz.com/'+input("请输入网址："),headers=h)print(response.status_code)html = response.text#print(html)#解析数据pattern = re.compile('class="MoreInfo".*?>(.*?)</p>',re.S)result = re.findall(pattern,html)# 方法一：# str = re.sub('n',',',result[0])# print(str)#方法二：print(result[0].replace('/n',','))

"""爬取*眼电影前100电影信息"""import requestsimport reimport time# count = [0,10,20,30,40,50,60,70,80,90]h = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36'}responce = requests.get('https://XX.com/board/4?offset=0', headers=h)responce.encoding = 'utf-8'html = responce.text# 解析数据  time.sleep(2)patter = re.compile('class="name">.*?title="(.*?)".*?主演：(.*?)</p>.*?上映时间：(.*?)</p>', re.S)#time.sleep(2)result = re.findall(patter, html)print(result)with open('maoyan.txt', 'a', encoding='utf-8') as f:for item in result:  # 读取result（以元组的形式储存）中的内容=》for i in item:f.write(i.strip().replace('n', ','))#print('n')

"""*精灵爬取练习 http://616pic.com/png/  ==》 http://XX.616pic.com/ys_img/00/06/20/64dXxVfv6k.jpg"""import requestsimport reimport time#数据的爬取img的urldef get_urls():response = requests.get('http://XX.com/png/')html_str = response.text#解析数据，得到urlpattern = re.compile('<img ')results = re.findall(pattern,html_str)print(results)return results#<img >#下载图片def down_load_img(urls):for url in urls:response = requests.get(url)with open('temp/'+url.split('/')[-1], 'wb') as f:f.write(response.content)print(url.split('/')[-1],'已经下载成功')if __name__ == '__main__':urls = get_urls()down_load_img(urls)

'''美女爬取====方法一'''import requestsimport reurl = 'https://www.toutiao.com/api/search/content/?aid=24&app_name=web_search&offset=0&format=json&keyword=%E7%BE%8E%E5%A5%B3&autoload=true&count=20&en_qc=1&cur_tab=1&from=search_tab&pd=synthesis&timestamp=1596180364628&_signature=-Bv0rgAgEBA-TE0juRclmfgatbAAKdC7s6ktYqc7u9jLqXOQ5SBCDkd25scxRvDydd6TgtOw0B7RVuaQxhwY1BwV89sPbdam8LkNuV08d0QfrZqQ4oOOrOukEJ1qxroigLT'response = requests.get(url)print(response.status_code)html_str = response.text#解析"large_image_url":"(.*?)"pattern = re.compile('"large_image_url":"(.*?)"')urls = re.findall(pattern,html_str)print(urls)
def down_load(urls):    for url in urls:        response = requests.get(url)        with open('pic/'+url.split('/')[-1],'wb') as f:            f.write(response.content)            print(url.split('/')[-1],'已经下载成功')if __name__ == '__main__':    down_load(urls)

'''美女爬取====方法二'''import requestsimport refrom urllib.parse import urlencode#https://www.toutiao.com/api/search/content/?aid=24&app_name=web_search&offset=0&format=json&keyword=%E7%BE%8E%E5%A5%B3&autoload=true&count=20
def get_urls(page):    keys = {        'aid':'24',        'app_name':'web_search',        'offset':20*page,        'keyword':'美女',        'count':'20'    }    keys_word = urlencode(keys)    url = 'https://www.toutiao.com/api/search/content/?'+keys_word    response = requests.get(url)    print(response.status_code)    html_str = response.text    # 解析"large_image_url":"(.*?)"    pattern = re.compile('"large_image_url":"(.*?)"',re.S)    urls = re.findall(pattern, html_str)    return urls
#下载图片def download_imags(urls):    for url in urls:        response = requests.get(url)        with open('pic/'+url.split('/')[-1]+'.jpg','wb') as f:            f.write(response.content)            print(url.split('/')[-1]+'.jpg',"已下载~~")
if __name__ == '__main__':    for page in range(3):        urls = get_urls(page)        print(urls)        download_imags(urls)

"""线程池"""
from concurrent.futures import ThreadPoolExecutorimport timeimport threading
def ban_zhuang(i):    print(threading.current_thread().name,"**开始搬砖{}**".format(i))    time.sleep(2)    print("**员工{}搬砖完成**一共搬砖：{}".format(i,12**2))   #将format里的内容输出到{}
if __name__ == '__main__':             #主线程    start_time = time.time()    print(threading.current_thread().name,"开始搬砖")    with ThreadPoolExecutor(max_workers=5) as pool:        for i in range(10):            p = pool.submit(ban_zhuang,i)    end_time =time.time()    print("一共搬砖{}秒".format(end_time-start_time))

'''美女爬取'''import requestsimport refrom urllib.parse import urlencodeimport time
import threading#https://www.toutiao.com/api/search/content/?aid=24&app_name=web_search&offset=0&format=json&keyword=%E7%BE%8E%E5%A5%B3&autoload=true&count=20
def get_urls(page):    keys = {        'aid':'24',        'app_name':'web_search',        'offset':20*page,        'keyword':'美女',        'count':'20'    }    keys_word = urlencode(keys)    url = 'https://www.toutiao.com/api/search/content/?'+keys_word    response = requests.get(url)    print(response.status_code)    html_str = response.text    # 解析"large_image_url":"(.*?)"    pattern = re.compile('"large_image_url":"(.*?)"',re.S)    urls = re.findall(pattern, html_str)    return urls
#下载图片def download_imags(urls):    for url in urls:        try:            response = requests.get(url)            with open('pic/'+url.split('/')[-1]+'.jpg','wb') as f:                f.write(response.content)                print(url.split('/')[-1]+'.jpg',"已下载~~")        except Exception as err:            print('An exception happened: ')

if __name__ == '__main__':    start = time.time()    thread = []    for page in range(3):        urls = get_urls(page)        #print(urls)        #多线程        for url in urls:            th = threading.Thread(target=download_imags,args=(url,))            #download_imags(urls)            thread.append(th)    for t in thread:        t.start()    for t in thread:        t.join()
    end = time.time()    print('耗时：',end-start)

User-agent: BaiduspiderDisallow: /baiduDisallow: /s?Disallow: /ulink?Disallow: /link?Disallow: /home/news/data/Disallow: /bhUser-agent: GooglebotDisallow: /baiduDisallow: /s?Disallow: /shifen/Disallow: /homepage/Disallow: /cproDisallow: /ulink?Disallow: /link?Disallow: /home/news/data/Disallow: /bh