scrapy startproject yok

cd yok
scrapy genspider pinglun yok.com

import scrapy
from scrapy.item import Item, Field
class YoukuItem(Item):
    content = Field()

import scrapy
import json
from yok.items import YokItem

class PinglunSpider(scrapy.Spider):
    name = 'pinglun'
    allowed_domains = ['yok.com']
    episodes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
    def start_requests(self):
        for ep in self.episodes:
            url = 'https://p.Id={0}&Page=1&pageSize=30'.format(str(ep))
            yield scrapy.Request(url=url, callback=self.parse_pages, meta={'ep':ep}, dont_filter=True)
    def parse_pages(self,response):
    　　'''获取总页数'''
        ep, results= response.meta['ep'], response.text
        results = results.replace('n_commentList(', '').replace(')', '')
        results = json.loads(results, strict=False)
        if results.get('code') == 0:
            pages = results.get('data').get('totalPage')
            for page in range(1, int(pages)+1):
                url = 'https://p.Id={0}Page={1}&pageSize=30'.format(str(ep), str(page))
                yield scrapy.Request(url=url, callback=self.parse, dont_filter=True)
    def parse(self, response):
    　　'''解析内容'''
        results = response.text
        results= results.replace('n_commentList(', '').replace(')', '')
        #strict表示对json语法要求不严格
        results = json.loads(results, strict = False)
        item = YoukuItem()
        if results.get('data'):
            if results.get('data').get('comment'):
                comments = results.get('data').get('comment')
                for comment in comments:
                    item['content'] =comment.get('content').strip().replace(r'\n','')yield item

import pymysql
class YokPipeline(object):
    def __init__(self):
        self.db = pymysql.connect(
            host='localhost',
            user='root',
            password='1234',
            db='yok',
            port=3306,
            charset='utf8'
        )
        self.cursor = self.db.cursor()
    def process_item(self, item, spider):
        sql = "INSERT INTO pinglun(content) values(%s)"
        data = (item["content"])
        try:
            self.cursor.execute(sql, data)
            self.db.commit()
        except:
            print('存储失败')
            self.db.rollback()
    def closed_spider(self, spider):
        self.cursor.close()
        self.db.close()

BOT_NAME = 'yok'
SPIDER_MODULES = ['yok.spiders']
NEWSPIDER_MODULE = 'yok.spiders'
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 2
COOKIES_ENABLED = False
DEFAULT_REQUEST_HEADERS = {
    ':authority': 'p.comments.yok.com',
    ':method': 'GET',':scheme': 'https',
    'accept': '*/*',
    'accept-encoding': 'gzip, deflate, br',
    'accept-language': 'zh-CN,zh;q=0.9',
    'referer': 'https://www.yok.com/id.html'
}
ITEM_PIPELINES = {
   'yok.pipelines.YokPipeline': 300,
}

from scrapy import cmdline

cmdline.execute('scrapy crawl pinglun'.split())

import pandas as pd
import pymysql
#连接数据库
conn = pymysql.connect(host='localhost', user='root', password='1234', port=3306, db='dn', charset='utf8')
sql = 'select * from pinglun'
#读取数据
df = pd.read_sql(sql, conn)
#删除重复记录
df.drop_duplicates(inplace = True)
#设置行的最大显示为2条
pd.set_option('display.max_rows', 2)
df

import numpy as np
#转换数据类型
df.score = df.score.astype(int)
teleplay_0 = df[df.score <3]
#通过比对,发现消极情感的总数较少,随机抽取同数量的积极情感
teleplay_1 = df[df.score > 3].sample(n=teleplay_0.shape[0])
#对两者进行拼接
teleplay = pd.concat([teleplay_1, teleplay_0], axis=0)
#对情感进行分类, 0消极,1积极
teleplay['emotion'] = np.where(teleplay.score > 3, 1, 0)
teleplay

pip install jieba
pip install snownlp

from snownlp import SnowNLP
import jieba
#对评论进行分词, 并以空格隔开
teleplay['cut_jieba'] = teleplay.content.apply(lambda x: ' '.join(jieba.cut(x)))
teleplay['cut_snownlp'] = teleplay.content.apply(lambda x: ' '.join(SnowNLP(x).words))
teleplay

def get_stopwords(path):
    '''读取停用词'''
    with open(path) as f:
        stopwords = [i.strip() for i in f.readlines()]
        return stopwords
    
path = r'D:\stopword.txt'
stopwords = get_stopwords(path)
#分别去除cut_jieba和cut_snownlp中的停用词
teleplay['cut_jieba'] = teleplay.cut_jieba.apply(lambda x: ' '.join([w for w in (x.split(' ')) if w not in stopwords]))
teleplay['cut_snownlp'] = teleplay.cut_snownlp.apply(lambda x: ' '.join([w for w in (x.split(' ')) if w not in stopwords]))
teleplay

from sklearn.model_selection import train_test_split
#分别对cut_jieba和cut_snownlp进行划分
X1, X2, y = teleplay[['cut_jieba']], teleplay[['cut_snownlp']], teleplay.emotion
X1_train, X1_test, y1_train, y1_test = train_test_split(X1, y, test_size=0.2)
X2_train, X2_test, y2_train, y2_test = train_test_split(X2, y, test_size=0.2)

from sklearn.feature_extraction.text import CountVectorizer
#初始化
vect = CountVectorizer()
#分别对cut_jieba和cut_snownlp进行向量化, 并转化为dataframe.
vect_matrix_1 = pd.DataFrame(vect.fit_transform(X1_train.cut_jieba).toarray(), columns = vect.get_feature_names())
vect_matrix_2 = pd.DataFrame(vect.fit_transform(X2_train.cut_snownlp).toarray(), columns = vect.get_feature_names())
vect_matrix_1
vect_matrix_2

#max_df的类型若为int, 则表示过滤掉文档中出现次数大于max_df的词汇, 若为float时, 则是百分比; min_df类似, token_pattern表示token的正则表达式
vect = CountVectorizer(max_df = 0.8, 
                     min_df = 2,
                     token_pattern = r"(?u)\b[^\d\W]\w+\b")
vect_matrix_1 = pd.DataFrame(vect.fit_transform(X1_train.cut_jieba).toarray(), columns = vect.get_feature_names())
vect_matrix_2 = pd.DataFrame(vect.fit_transform(X2_train.cut_snownlp).toarray(), columns = vect.get_feature_names())
vect_matrix_1
vect_matrix_2

from sklearn.pipeline import make_pipeline
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import cross_val_score
#初始化朴素贝叶斯模型
nb = MultinomialNB()
#利用pipeline管道进行特征向量化
pipe = make_pipeline(vect, nb)
#交叉验证,将训练集分成10份,即计算10次,在每次计算中用10份中的一份当作验证集,对应的另外9份用作训练集,最后对10次计算出的准确率求平均值
score1 = cross_val_score(pipe, X1_train.cut_jieba, y1_train, cv=10, scoring='accuracy').mean()
score2 = cross_val_score(pipe, X2_train.cut_snownlp, y2_train, cv=10, scoring='accuracy').mean()
score1
score2

from sklearn import metrics
#训练出模型
pipe1= pipe.fit(X1_train.cut_jieba, y1_train)
pipe2= pipe.fit(X2_train.cut_jieba, y2_train)
#用训练的模型分别预测cut_jieba和cut_snownlp的测试集
y1_pre =  pipe1.predict(X1_test.cut_jieba)
y2_pre =  pipe2.predict(X2_test.cut_snownlp)
#评价模型预测结果
metrics.accuracy_score(y1_test, y1_pre)
metrics.accuracy_score(y2_test, y2_pre)

#分别求得cut_jieba和cut_snownlp的混淆矩阵
con_matrix1 = metrics.confusion_matrix(y1_test, y1_pre)
con_matrix2 = metrics.confusion_matrix(y2_test, y2_pre)
#分别计算精准率和召回率
accu_rate1, accu_rate2= con_matrix1[0][0]/(con_matrix1[1][0]+con_matrix1[0][0]), con_matrix2[0][0]/(con_matrix2[1][0]+con_matrix2[0][0])
recall_rate1, recall_rate2 = con_matrix1[0][0]/(con_matrix1[0][1]+con_matrix1[0][0]), con_matrix2[0][0]/(con_matrix2[0][1]+con_matrix2[0][0])
print('jieba_confused_matrix:')
print(con_matrix1)
print('accurate_rate:{0}, recall_rate:{1}'.format(accu_rate1, recall_rate1))
print()
print('snownlp_confused_matrix:')
print(con_matrix2)
print('accurate_rate:{0}, recall_rate:{1}'.format(accu_rate2, recall_rate2))

y_pred_snow = X1_test.cut_jieba.apply(lambda x: SnowNLP(x).sentiments)
y_pred_snow= np.where(y_pred_snow > 0.5, 1, 0)
metrics.accuracy_score(y1_test, y_pred_snow)
metrics.confusion_matrix(y1_test, y_pred_snow)

#读取数据, 删除重复项以及jieba分词
conn = pymysql.connect(host = 'localhost', user='root', password = '1234', port=3306,db='yok', charset='utf8')
sql = 'select * from pinglun'
yk = pd.read_sql(sql, conn)
yk.drop_duplicates(inplace=True)
yk['cut_content'] = yk.content.apply(lambda x: ' '.join(jieba.cut(x)))
#用模型进行情感分类
yk['emotion'] = pipe1.predict(yk.cut_content)
#计算积极情感占比
yk[yk.emotion_nb == 1].shape[0]/yk.shape[0]

#利用snownlp进行情感分类
emotion_pred = yk.cut_content.apply(lambda x: SnowNLP(x).sentiments)
yk['emotion_snow'] = np.where(emotion_pred > 0.5, 1, 0)
#计算积极情感占比
yk[yk.emotion_snow == 1].shape[0]/yk.shape[0]

#随机抽取1个
yk.sample(n=1)

import matplotlib.pyplot as plt
from wordcloud import WordCloud
bg_image = plt.imread(r'tig.jpg')                
wc = WordCloud(width=1080,                       #设置宽度
               height=840,                       #设置高度
               background_color='white',         #背景颜色
               mask=bg_image,                    #背景图
               font_path='STKAITI.TTF',          #中文字体
               stopwords=stopwords,              #停用词
               max_font_size=400,                #字体最大值
               random_state=50                   #随机配色方案
              )            
wc.generate_from_text(' '.join(yk.cut_content))  #生成词云
plt.imshow(wc)                                   # 绘制图像
plt.axis('off')                                  # 关闭坐标轴显示
wc.to_file('pig.jpg')
plt.show()

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import metrics
vect = TfidfVectorizer(max_df = 0.8, 
                     min_df = 2,
                     token_pattern = r"(?u)\b[^\d\W]\w+\b")
nb = MultinomialNB()
pipe = make_pipeline(vect, nb)
#训练模型
pipe.fit(X_train.cut_jieba, y_train)
#对测试数据进行预测
y_pre =  pipe.predict(X_test.cut_jieba)
#训练数据集分数
print(cross_val_score(pipe, X_train.cut_jieba, y_train, cv=10, scoring='accuracy').mean())
#测试数据分数
print(metrics.accuracy_score(y_test, y_pre))
#混淆矩阵
con_matrix = metrics.confusion_matrix(y_test, y_pre)
#精准度
con_matrix[0][0]/(con_matrix[1][0]+con_matrix[0][0])
#召回率
con_matrix[0][0]/(con_matrix[0][1]+con_matrix[0][0])