python爬虫--下载煎蛋网妹子图到本地

 1 # -*- coding:utf-8 -*-
 2 """
 3 下载煎蛋妹子到本地，通过selenium、正则表达式、phantomjs、Beautifulsoup实现
 4 """
 5 
 6 import re
 7 import os
 8 
 9 from selenium import webdriver
10 from selenium.webdriver.support.wait import WebDriverWait
11 from selenium.webdriver.support import expected_conditions as EC
12 from selenium.webdriver.common.by import By
13 from selenium.common.exceptions import TimeoutException
14 from bs4 import BeautifulSoup
15 from urllib import urlretrieve
16 
17 #解决谷歌浏览器正受到自动测试软件的控制
18 # options = webdriver.ChromeOptions()
19 # options.add_argument('disable-infobars')
20 
21 url = 'http://jandan.net/ooxx'
22 # driver = webdriver.Chrome(chrome_options=options)
23 driver = webdriver.PhantomJS()
24 wait = WebDriverWait(driver, 30)
25 
26 #下载的煎蛋妹子保存的文件夹
27 img_save_file = 'images'
28 
29 #获取总页数。打开煎蛋网-妹子图默认页面可以获取到总页数
30 def get_default_page_num():
31     try:
32         driver.get(url)
33         page_element = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.current-comment-page')))
34         return page_element.text
35     except TimeoutException:
36         get_default_page_num()
37 
38 #获取图片的url
39 def get_img_url(page_number):
40     img_url_list = []
41     url = r'http://jandan.net/ooxx/page-'+ str(page_number) + r'#comments'
42     print url
43     # url = 'http://www.baidu.com'
44     html = driver.get(url)
45     try:
46         driver.get(url)
47         wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#comments > ol img')))
48     except TimeoutException:
49         print "打开页面失败，重新加载该页面"
50         get_img_url(page_number)
51 
52     #获取页面html元素
53     html = driver.page_source
54     #通过BeautifulSoup解析
55     soup = BeautifulSoup(html, 'html.parser')
56     #找出所有为img的标签
57     imgs = soup.find_all('img')
58     #gif图片需要获取ora_src属性，才是完整的gif图片。has_attr 判断是否有某个属性，attrs可以获取属性值
59     for img in imgs:
60         if img.has_attr('org_src'):
61             img_url = img.attrs['org_src']
62         else:
63             img_url = img.attrs['src']
64         img_url_list.append(img_url)
65     return img_url_list
66 
67 #下载图片，通过urllib的urlretrieve实现
68 def download_img(img_url):
69     img_name = img_url.split('/')[-1]
70     img_save_path = img_save_file + '/' +img_name
71     urlretrieve(img_url, img_save_file + '/' + img_name)
72 
73 #创建图片存储所在的文件夹
74 def add_img_save_file(img_save_file):
75     if os.path.exists(img_save_file):
76         pass
77     else:
78         os.makedirs(img_save_file)
79 
80 def main():
81     add_img_save_file(img_save_file)
82     #通过正则表达式提取当前的页数
83     partner = re.compile(r'(\d+)')
84     content = get_default_page_num()
85     total_pages = partner.search(content).group()
86 
87     for i in range(1, int(total_pages) + 1):
88         print "正在下载第" + str(i) + '的图片，url为：',
89         img_url_list = get_img_url(str(i))
90         for img_url in img_url_list:
91             download_img(img_url)
92 
93 if __name__ == '__main__':
94     main()