Python+Scrapy爬取腾讯新闻首页所有新闻及评论

# -*- coding: utf-8 -*-
from scrapy.spiders import Spider  
from scrapy.http import Request  
from scrapy.selector import Selector  
from test1.items import NewsItem,ListCombiner
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
import requests
import re
import json

class TencentNewsSpider(CrawlSpider):
    name = 'tencent_news_spider'
    allowed_domains = ['new.qq.com','news.qq.com']
    start_urls = [
        'http://news.qq.com'
    ]
    url_pattern1= r'(.*)/a/(\d{8})/(\d+)\.htm'
    url_pattern2=r'(.*)/omn/(.+)\.html'
    url_pattern3=r'(.*)/omn/([A-Z0-9]{16,19})'
    url_pattern4=r'(.*)/omn/(\d{8})/(.+)\.html'
    rules = (
        Rule(LinkExtractor(allow=(url_pattern1)),'parse_news1'),
        Rule(LinkExtractor(allow=(url_pattern2)),'parse_news2'),
        Rule(LinkExtractor(allow=(url_pattern3)),'parse_news3'),
    )

    def parse_news1(self, response):
        sel = Selector(response)
        print(response.url)
        pattern = re.match(self.url_pattern1, str(response.url))
        item = NewsItem()
        item['source'] = 'tencent'#pattern.group(1)
        item['date'] = pattern.group(2)
        item['newsId'] = pattern.group(3)
        item['cmtId'] = (sel.re(r"cmt_id = (.*);"))[0] # unicode string需要判断有没有cmtId，因为页面有可能为空
        item['comments'] = {'link':str('http://coral.qq.com/')+item['cmtId']}
        item['contents'] = {'link':str(response.url), 'title':u'', 'passage':u''}
        item['contents']['title'] = sel.xpath('//h1/text()').extract()[0]
        item['contents']['passage'] = ListCombiner(sel.xpath('//p/text()').extract())
        return item


    def parse_news2(self,response):
        sel = Selector(response)
        pattern = re.match(self.url_pattern4, str(response.url))
        item=NewsItem()
        item['source'] = 'tencent'#pattern.group(1)
        item['date'] = pattern.group(2)
        item['newsId'] = pattern.group(3)
        item['cmtId'] = (sel.re(r"\"comment_id\":\"(\d*)\","))[0]
        item['comments'] = {'link':str('http://coral.qq.com/')+item['cmtId']}
        item['contents'] = {'link':str(response.url), 'title':u'', 'passage':u''}
        item['contents']['title'] = sel.xpath('//h1/text()').extract()[0]
        item['contents']['passage'] = ListCombiner(sel.xpath('//p/text()').extract())
        return item

    def parse_news3(self,response):
        item = NewsItem()
        print(response.url)
        str1='http://openapi.inews.qq.com/getQQNewsNormalContent?id='
        str2='&chlid=news_rss&refer=mobilewwwqqcom&otype=jsonp&ext_data=all&srcfrom=newsapp&callback=getNewsContentOnlyOutput'
        pattern = re.match(self.url_pattern3, str(response.url))
        date=re.search(r"(\d{8})",pattern.group(2))#匹配时间
        item['source'] = 'tencent'#pattern.group(1)
        item['date'] = date.group(0)
        item['newsId'] = pattern.group(2)
        print(pattern.group(2))
        headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
        }
        out=self.getHTMLText(str1+pattern.group(2)+str2,headers)
        g=re.search("getNewsContentOnlyOutput\\((.+)\\)", out)
        out=json.loads(g.group(1))
        item['cmtId'] =out["cid"]
        item['comments'] = {'link':str('http://coral.qq.com/')+item['cmtId']}
        item['contents'] = {'link':str(response.url), 'title':u'', 'passage':u''}
        item['contents']['title'] = out["title"]
        item['contents']['passage'] =out["ext_data"]["cnt_html"]
        return item

    def getHTMLText(self,url,headers):
        try:
            r=requests.get(url, headers=headers)
            r.raise_for_status()
            r.encoding=r.apparent_encoding
            return r.text
        except:
            print("产生异常")

# -*- coding: utf-8 -*-
import requests
import re
import json
import codecs
import os
import datetime

# 爬取新闻评论id为commentid，日期为date，新闻id为newsID的所有评论
def crawlcomment(commentid,date,newsID):
    url1='http://coral.qq.com/article/'+commentid+'/comment/v2?callback=_articlecommentv2&orinum=30&oriorder=t&pageflag=1&cursor='
    url2='&orirepnum=10&_=1522383466213'
    # 一定要加头要不然无法访问
    headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
    }

    dir=os.getcwd();
    comments_file_path=dir+'/docs/tencent/' + date+'/'+newsID+'_comments.json'

    news_file = codecs.open(comments_file_path, 'a', 'utf-8')
    response=getHTMLText(url1+'0'+url2,headers)

    while 1:
        g=re.search("_articlecommentv2\\((.+)\\)", response)
        out=json.loads(g.group(1))
        if not out["data"]["last"]:
            news_file.close()
            print("finish！")
            break;
        for i in out["data"]["oriCommList"]:
            time=str(datetime.datetime.fromtimestamp(int(i["time"])))#将unix时间戳转化为正常时间
            line = json.dumps(time+':'+i["content"],ensure_ascii=False)+'\n'
            news_file.write(line)
            print(i["content"])

        url=url1+out["data"]["last"]+url2#得到下一个评论页面链接
        print(url)
        response=getHTMLText(url,headers)

def getHTMLText(url,headers):
    try:
        r=requests.get(url, headers=headers)
        r.raise_for_status()
        r.encoding=r.apparent_encoding
        return r.text
    except:
        return "产生异常 "