python小白的初步爬虫

# -*- coding: UTF-8 -*-

import xlwt
import urllib2
import json

# 设置编码格式为utf-8
import sys
reload(sys)
sys.setdefaultencoding('utf8')


# 设置url
shop_url = "https://youpin.mi.com/app/shopv3/pipe"

# 代理
user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
# 请求格式
con_type = 'application/json'
# 类目的referer
caty_referer = "https://youpin.mi.com/"
# 商品列表的referer
goods_referer = 'https://youpin.mi.com/detail?gid=101975'

# 组建header
caty_headers = {'Referer': caty_referer, "Content-Type": con_type}

# 分类入参
caty_data = {"result": {"model": "Homepage", "action": "GetGroup2ClassInfo", "parameters": {}}}
# 商品列表入参
list_data = {"uClassList": {"model": "Homepage", "action": "BuildHome", "parameters": {"id": 1}}}
# 评论入参
comment_data = {"overView": {"model": "Product", "action": "CommentIndexV2", "parameters": {"gid": 1}}}


# 获取商品的评论信息
def get_good_comment(url, gid):
    caty_headers["Referer"] = url
    comment_data["overView"]["parameters"]["gid"] = gid
    comment_param = json.dumps(comment_data)

    com_req = urllib2.Request(shop_url, headers=caty_headers, data=comment_param)
    com_resp = urllib2.urlopen(com_req)
    com_res = com_resp.read()
    com_result = json.loads(com_res)
    com_info = com_result['result']['overView']['data']
    com_count = 0

    if com_info['tags']:
        com_count = com_info['tags'][0]['count']
    else:
        com_count = 0
    # for com in com_info['tags']:
    #     com_count = com['count']

    comment_info = {
        "score": com_info['avg_score'],
        "rate": com_info['positive_rate'],
        "count": com_count
    }
    return comment_info


# 获取商品列表，并获取到商品信息，保存到excel 
def get_good_list(caty_id, caty_name):
    # 1. 请求分类的商品列表
    list_referer = "https://youpin.mi.com/goodsbycategory?firstId=%s&secondId=%s&title=%s"
    list_referer_r = list_referer % (caty_id, caty_id, caty_name)
    caty_headers["Referer"] = list_referer_r
    list_data["uClassList"]["parameters"]["id"] = caty_id
    list_param = json.dumps(list_data)

    list_req = urllib2.Request(shop_url, headers=caty_headers, data=list_param)
    list_resp = urllib2.urlopen(list_req)
    list_res = list_resp.read()
    list_result = json.loads(list_res)
    list_info = list_result['result']['uClassList']['data']

    worksheet = workbook.add_sheet(caty_name)
    worksheet.write(0, 0, '类目')
    worksheet.write(0, 1, '链接')
    worksheet.write(0, 2, '商品名称')
    worksheet.write(0, 3, '价格')
    worksheet.write(0, 4, '划线价')
    worksheet.write(0, 5, '评论星级')
    worksheet.write(0, 6, '评论满意度')
    worksheet.write(0, 7, '评论条数')

    # 3. 获取商品url，从而获取商品信息
    index = 1
    for list_index in list_info:
        if list_index['item_type'] == 5:
            continue

        # 获取商品url，获取商品信息
        child_caty_name = list_index['content']['name']
        real_caty_name = caty_name+'/'+child_caty_name

        for goods in list_index['data']:
            # 获取评论
            comment = get_good_comment(goods['url'], goods['gid'])

            # 获取商品信息
            goods_info = {
                "caty": real_caty_name,
                "url": goods['url'],
                "name": goods['name'],
                "price_min": goods['price_min'],
                "market_price": goods['market_price'],
                "score": comment['score'],
                "rate": comment['rate'],
                "count": comment['count']
            }
            print goods_info
            worksheet.write(index, 0, goods_info['caty'])
            worksheet.write(index, 1, goods_info['url'])
            worksheet.write(index, 2, goods_info['name'])
            worksheet.write(index, 3, goods_info['price_min'])
            worksheet.write(index, 4, goods_info['market_price'])
            worksheet.write(index, 5, goods_info['score'])
            worksheet.write(index, 6, goods_info['rate'])
            worksheet.write(index, 7, goods_info['count'])
            index = index + 1

    workbook.save(filename)
    return


# 入口

# 1. 类目入参处理,并请求获取类目信息列表
caty_param = json.dumps(caty_data)
req = urllib2.Request(shop_url, headers=caty_headers, data=caty_param)
response = urllib2.urlopen(req)
res = response.read()
caty_resp = json.loads(res)
caty_groups = caty_resp["result"]["result"]["data"]["groups"]
print len(caty_groups)


# 2. excel表格初始化
filename = "小米优品.xls".decode(encoding="utf-8")
workbook = xlwt.Workbook(encoding='utf-8')

# 3. 获取分类列表,以及id
for caty in caty_groups:
    for caty_class in caty:
        caty_info = {
            'id': caty_class['class']['ucid'],
            'name': caty_class['class']['name']
        }

        list_data["uClassList"]["parameters"]["id"] = caty_info["id"]
        # 获取该类目下的商品列表
        get_good_list(caty_info['id'], caty_info['name'])