mitmproxy实战-通过mitmdump爬取京东金榜排行数据

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

import json

def response(flow):
    url = 'https://api.m.jd.com/client.action'
    if flow.request.url.startswith(url):
        text = flow.response.text
        json_data = json.loads(text)
        #print(json_data)
        #获取金榜排行
        if ('result' in json_data) and ('tabs' in json_data['result']):
            getMainGoldRank(json_data)
        #获取金榜排行商品列表
        if ('result' in json_data) and ('mainRank' in json_data['result']):
            getJDProductInfo(json_data)
    #获取商品信息
    if flow.request.url.startswith(url):
        text = flow.response.text
        json_data = json.loads(text)
        if ('floors' in json_data) and len(json_data['floors'])>1:
            prod_skuId=json_data['floors'][0].get('data').get('extMap').get('skuId')
            prod_introduceInfo=text
            print('-------prod_skuId-------:+++',prod_skuId)
            print(prod_skuId, prod_introduceInfo)


# 获取金榜列表
def getMainGoldRank(json_data):
    tabs = json_data['result']['tabs']
    for tab in tabs:
        tabName = tab['tabName']  # 榜单名称
        tabData = tab['data']  # 榜单数据
        print(type(tabData))
        if type(tabData) is list:
            for rankdata in tabData:
                rankId = rankdata['id']
                rankTitle = rankdata['name']
                rankType = rankdata['rankType']
                skuId=None
                print(tabName, rankId, rankType, rankTitle, skuId)
        if type(tabData) is dict:
            relatedAdvIds = tab['relatedAdvIds']
            if type(relatedAdvIds) is list:
                for relatedAdvId in relatedAdvIds:
                    rankList = tabData[relatedAdvId]['rankList']
                    for rankdata in rankList:
                        rankId = rankdata['rankId']
                        rankTitle = rankdata['rankType']
                        rankType = rankdata['rankTitle']
                        skuId = rankdata['skuId']
                        print(tabName, rankId, rankType, rankTitle, skuId)

#获取JD金榜商品概要信息
def getJDProductInfo(json_data):
    #if ('result' in json_data) and ('mainRank' in json_data['result']):
    print(json_data)
    try:
        products=json_data['result']['mainRank']['products']
        prod_main_rank=json_data['result']['mainRank']
        prod_rankId=prod_main_rank.get('id')
        #判断如果没有在main_gold_rank表中则添加记录至main_gold_rank表
        rankTitle=prod_main_rank.get('name')
        print('', prod_rankId,rankTitle , 13, '')
        for product in products:
            prod_skuId = product['product']['skuId']
            prod_name = product['product']['name']
            prod_img = product['product']['img']
            prod_skuSallingPoint = product.get('skuSallingPoint')
            prod_saleInfoStr = product.get('saleInfoStr')
            prod_simpleSaleInfoStr = product.get('simpleSaleInfoStr')
            prod_totalBuyInfoStr = product.get('totalBuyInfoStr')
            prod_goodCountStr = product.get('goodCountStr')
            prod_simpleGoodCountStr = product.get('simpleGoodCountStr')
            prod_totalPopularity = product.get('totalPopularity')
            prod_popularityStr = product.get('popularityStr')
            prod_cmttTag = product.get('cmttTag')
            prod_longTitle = product.get('longTitle')
            prod_authorityScore = product.get('authorityScore')
            prod_saleScore = product.get('saleScore')
            prod_popularityScore = product.get('popularityScore')
            prod_growthScore = product.get('growthScore')
            prod_praiseScore = product.get('praiseScore')
            prod_brandScore = product.get('brandScore')
            prod_brandStr = product.get('brandStr')
            prod_growthStr = product.get('growthStr')
            prod_sortedSaleIfoStr = product.get('sortedSaleIfoStr')
            prod_rankNum = product.get('rankNum')
            print(prod_rankId,prod_skuId,prod_name,prod_img,prod_skuSallingPoint,prod_saleInfoStr,prod_simpleSaleInfoStr,prod_totalBuyInfoStr,
            prod_goodCountStr,prod_simpleGoodCountStr,prod_totalPopularity,prod_popularityStr,prod_cmttTag,prod_longTitle,
            prod_authorityScore,prod_saleScore,prod_popularityScore,prod_growthScore,prod_praiseScore,prod_brandScore,prod_brandStr,prod_growthStr,prod_sortedSaleIfoStr,prod_rankNum)
    except Exception as e:
        print('getJDProductInfo Error:', e)