Python项目--Scrapy框架(一)

pip install Scrapy

scrapy startproject JianShu

cd JianShu
scrapy genspider JianShuSpider JianShuSpider.toscrape.com

 1 import scrapy
 2 from scrapy.item import Item, Field
 3 
 4 class JianshuItem(Item):
 5     # define the fields for your item here like:
 6     # name = scrapy.Field()
 7     title = Field()         #主题
 8     content  = Field()      #内容
 9     article = Field()       #文章
10     fans = Field()          #粉丝

 1 import scrapy
 2 from scrapy.spiders import CrawlSpider
 3 from scrapy.selector import Selector
 4 from JianShu.items import JianshuItem
 5 from scrapy.http import Request
 6 class JianShu(CrawlSpider):
 7     name = 'JianShu'
 8     allowed_domains = ['JianShuSpider.toscrape.com']
 9     start_urls = ['https://www.jianshu.com/recommendations/collections?page=1&order_by=hot']
10     def parse(self, response):
11         item = JianshuItem()
12         #对源码进行初始化
13         selector = Selector(response)
14         #采用xpath进行解析
15         infos = selector.xpath('//div[@class="collection-wrap"]')
16         for info in infos:
17             title = info.xpath('a[1]/h4/text()').extract()[0]
18             content = info.xpath('a[1]/p/text()').extract()
19             article = info.xpath('div/a/text()').extract()[0]
20             fans = info.xpath('div/text()').extract()[0]
21             #加入判断, 如果content存在则返回content[0], 否则返回''
22             if content:
23                 content = content[0]
24             else:
25                 content = ''
26             item['title'] = title
27             item['content'] = content
28             item['article'] = article
29             item['fans'] = fans
30             yield item
31         #列表生成式, 生成多个url
32         urls = ['https://www.jianshu.com/recommendations/collections?page={0}&order_by=hot'.format(str(page)) for page in range(2,37)]
33         for url in urls:
34             yield Request(url,callback=self.parse)

 1 import pymongo
 2 
 3 class JianshuPipeline(object):
 4     def __init__(self):
 5         '''连接Mongodb'''
 6         client = pymongo.MongoClient(host='localhost')
 7         db = client['test']
 8         jianshu = db["jianshu"]
 9         self.post = jianshu
10     def process_item(self, item, spider):
11         '''写入Mongodb'''
12         info = dict(item)
13         self.post.insert(info)
14         return item

 1 BOT_NAME = 'JianShu'
 2 SPIDER_MODULES = ['JianShu.spiders']
 3 NEWSPIDER_MODULE = 'JianShu.spiders'
 4 #从网站请求头复制粘贴User-Agent
 5 USER_AGENT = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
 6 ROBOTSTXT_OBEY = True
 7 #设置等待时间5秒
 8 DOWNLOAD_DELAY = 5
 9 #配置项目管道
10 ITEM_PIPELINES = {
11    'JianShu.pipelines.JianshuPipeline': 300,
12 }

1 from scrapy import cmdline
2 cmdline.execute('scrapy crawl JianShu'.split())