Scrapy持久化存储

    scrapy crawl 爬虫名称 -o xxx.json
    scrapy crawl 爬虫名称 -o xxx.xml
    scrapy crawl 爬虫名称 -o xxx.csv

ITEM_PIPELINES = {
    'qiubaiPro.pipelines.QiubaiproPipelineByRedis': 300,
}

# -*- coding: utf-8 -*-
import scrapy


class QiubaiSpider(scrapy.Spider):
    name = 'qiubai'
    # allowed_domains = ['www.qiushibaike.com']
    start_urls = ['https://www.qiushibaike.com/text/']

    def parse(self, response):
        div_list = response.xpath('//div[@id="content-left"]/div')
        all_data = []
        # xpath返回的列表元素类型为Selector类型
        for div in div_list:
            # title = div.xpath('./div[1]/a[2]/h2/text() | ./div[1]/span[2]/h2/text()')[0].extract()
            author = div.xpath('./div[1]/a[2]/h2/text() | ./div[1]/span[2]/h2/text()').extract_first()
            content = div.xpath('./a[1]/div/span/text()').extract_first()

            dic = {
                'author': author,
                'content': content
            }

            all_data.append(dic)
        # 基于终端指令的持久化存储:可以通过终端指令的形式将parse方法的返回值中存储的数据进行本地磁盘的持久化存储
        return all_data

BOT_NAME = 'qiubaiPro'
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
SPIDER_MODULES = ['qiubaiPro.spiders']
NEWSPIDER_MODULE = 'qiubaiPro.spiders'
ROBOTSTXT_OBEY = False

scrapy crawl qiubai -o qiubai.csv

# -*- coding: utf-8 -*-
import scrapy
from bossPro.items import BossproItem


class BossSpider(scrapy.Spider):
    name = 'boss'
    allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.zhipin.com/job_detail/?query=Python爬虫&scity=101010100&industry=&position=']

    def parse(self, response):
        li_list = response.xpath('//div[@class="job-list"]/ul/li')
        for li in li_list:
            title = li.xpath('.//div[@class="info-primary"]/h3[@class="name"]/a/div/text()').extract_first()
            salary = li.xpath('.//div[@class="info-primary"]/h3[@class="name"]/a/span/text()').extract_first()
            company = li.xpath('.//div[@class="company-text"]/h3/a/text()').extract_first()

            # 实例化一个item类型的对象
            item = BossproItem()
            # 将解析到的数据存储到item对象中
            item["title"] = title
            item["salary"] = salary
            item["company"] = company

            # 将item对象提交给管道进行持久化存储
            yield item

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class BossproItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    salary = scrapy.Field()
    company = scrapy.Field()

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

# 管道文件:需要接收爬虫文件提交过来的数据,并对数据进行持久化存储.(IO操作)
class BossproPipeline(object):
    fp = None
    # 只会被执行一次(开始爬虫的时候执行一次)
    def open_spider(self,spider):
        print("开始爬虫")
        self.fp = open('./job.txt','w',encoding='utf-8')
    # 爬虫文件每提交一次,该方法就会被调用一次
    def process_item(self, item, spider): #300表示为优先级，值越小优先级越高
        self.fp.write(item['title'] + "\t" + item['salary'] + '\t' + item['company'] + '\n')
        return item
    # 结束爬虫时执行
    def close_spider(self,spider):
        self.fp.close()
        print("爬虫结束")

# 注意:默认情况下,管道机制并没有开启,需要手动在配置文件中进行开启

# 使用管道进行持久化的流程:
#   1.获取解析到的数据
#   2.将解析的数据存储到item对象(item类中进行相关属性的声明)
#   3.通过yield关键字将item提交到管道
#   4.管道文件中进行持久化存储代码的编写(process_item)
#   5.在配置文件中开启管道

#开启管道
ITEM_PIPELINES = {
    'secondblood.pipelines.SecondbloodPipeline': 300, #300表示为优先级，值越小优先级越高
}

scrapy crawl boss --nolog

import pymysql
class mysqlPipeline(object):
    conn = None
    cursor = None
    def open_spider(self,spider):
        self.conn = pymysql.Connect(host='127.0.0.1', port=3306, user='root', password='', db='spider')
        print(self.conn)
    def process_item(self, item, spider):
        self.cursor = self.conn.cursor()
        sql = 'insert into boss values("%s","%s","%s")'%(item['title'],item['salary'],item['company'])
        try:
            self.cursor.execute(sql)
            self.conn.commit()
        except Exception as e:
            print (e)
            self.conn.rollback()

    def close_spider(self,spider):
        self.cursor.close()
        self.conn.close()

# 开启管道,自定义管道向不用的数据库存储数据
# 300是优先级,数字越小,优先级越高
 
ITEM_PIPELINES = {
   'boss.pipelines.BossPipeline': 300,
   'boss.pipelines.mysqlPipeLine': 301,
}

from redis import Redis
class RedisPipeline(object):
    conn = None
    def process_item(self,item,spider):
        dic = {
            "title":item["title"],
            "salary":item["salary"],
            "company":item["company"]
        }
        self.conn.lpush("jobInfo",json.dumps(dic))
    def open_spider(self,spider):
        self.conn = Redis(host='127.0.0.1',port=6379)
        print (self.conn)

ITEM_PIPELINES = {
   #'bossPro.pipelines.BossproPipeline': 300,
   #'bossPro.pipelines.mysqlPipeline': 301,
   'bossPro.pipelines.RedisPipeline': 302,
}