爬取网易新闻排行榜

import os
import sys
import requests
import re

from lxml import etree


def StringListSave(save_path, filename, slist):
    # 检测是否有这个文件目录，不存在的话，会自动创建
    if not os.path.exists(save_path):
        os.makedirs(save_path)
    path = save_path+"/"+filename+".txt"
    with open(path, "w+") as fp:
        for s in slist:
            # 做了utf8转码,转为终端可识别的码制
            fp.write("%s\t\t%s\n" % (s[0].encode("utf8").decode('utf-8'), s[1].encode("utf8").decode('utf-8')))


def Page_Info(myPage):
    '''Regex'''
    # 这里的re.findall 返回的是一个元组列表,内容是 (.*?) 中匹配到的内容
    # 析取每个链接的标题和链接
    mypage_Info = re.findall(r'<div class="titleBar" id=".*?"><h2>(.*?)'
                             r'</h2><div class="more"><a href="(.*?)">.*?</a></div></div>', myPage, re.S)
    return mypage_Info

def New_Page_Info(new_page):
    '''Regex(slowly) or Xpath(fast)'''
    # 将new_page的内容转为html格式的树
    dom = etree.HTML(new_page)
    # 析取 <tr <td <a中的文本
    new_items = dom.xpath('//tr/td/a/text()')
    # 析取 <tr <td <a中的链接, @href 是一个属性
    new_urls = dom.xpath('//tr/td/a/@href')
    assert(len(new_items) == len(new_urls))
    return zip(new_items, new_urls)

def Spider(url):
    i = 0
    print("downloading ", url)
    myPage = requests.get(url).content.decode("gbk")
    myPageResults = Page_Info(myPage)
    save_path = "网易新闻抓取"
    filename = str(i)+"_"+"新闻排行榜"
    StringListSave(save_path, filename, myPageResults)
    i += 1
    for item, url in myPageResults:
        print("downloading ", url)
        new_page = requests.get(url).content.decode("gbk")
        newPageResults = New_Page_Info(new_page)
        filename = str(i)+"_"+item
        StringListSave(save_path, filename, newPageResults)
        i += 1


if __name__ == '__main__':
    print("start")
    start_url = "http://news.163.com/rank/"
    Spider(start_url)
    print("end")

import os
save_path = "网易新闻抓取"
if not os.path.exists(save_path):
   os.makedirs(save_path)

import re
with open('/rr.txt', 'w') as f:
   f.write('hello world')

import re
with open('/rr.txt', 'a') as f:
    f.write('hello world\n')
    # print(f.readline(1))

%s\t\t%s\n