爬虫：输入网页之后爬取当前页面的图片和背景图…

# -*- coding:utf-8 -*-#

# filename: prt_cmd_color.py

import ctypes, sys

STD_INPUT_HANDLE = -10
STD_OUTPUT_HANDLE = -11
STD_ERROR_HANDLE = -12

# 字体颜色定义 text colors
FOREGROUND_BLUE = 0x09  # blue.
FOREGROUND_GREEN = 0x0a  # green.
FOREGROUND_RED = 0x0c  # red.
FOREGROUND_YELLOW = 0x0e  # yellow.

# 背景颜色定义 background colors
BACKGROUND_YELLOW = 0xe0  # yellow.

# get handle
std_out_handle = ctypes.windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE)


def set_cmd_text_color(color, handle=std_out_handle):
    Bool = ctypes.windll.kernel32.SetConsoleTextAttribute(handle, color)
    return Bool


# reset white
def resetColor():
    set_cmd_text_color(FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE)


# green
def printGreen(mess):
    set_cmd_text_color(FOREGROUND_GREEN)
    sys.stdout.write(mess)
    resetColor()


# red
def printRed(mess):
    set_cmd_text_color(FOREGROUND_RED)
    sys.stdout.write(mess)
    resetColor()


# yellow
def printYellow(mess):
    set_cmd_text_color(FOREGROUND_YELLOW)
    sys.stdout.write(mess + '\n')
    resetColor()


# white bkground and black text
def printYellowRed(mess):
    set_cmd_text_color(BACKGROUND_YELLOW | FOREGROUND_RED)
    sys.stdout.write(mess + '\n')
    resetColor()

if __name__ == '__main__':
    printGreen('printGreen:Gree Color Text')
    printRed('printRed:Red Color Text')
    printYellow('printYellow:Yellow Color Text')

# -*- coding: utf-8 -*-
## import some modules
import os
import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup
from os import  path
import requests
import re
from urllib.parse import urlparse, urljoin
from colors import  *
d = path.dirname(__file__)
bar_length = 20
def output(List, percent, msg ,url):
    hashes = '#' * int(percent / len(List) * bar_length)
    spaces = ' ' * (bar_length - len(hashes))
    loadingStr = str(int(100 * percent / len(List)))+ u'%'
    length = len('100%')
    if len(loadingStr) < length:
        loadingStr += ' '*(length-len(loadingStr))
    sys.stdout.write("\rPercent: [%s %s]" % (hashes + spaces, loadingStr ))
    printYellow("         [%s] %s " % ( msg, url))
    sys.stdout.flush()
    time.sleep(2)

class Spider():
    '''spider class'''
    def __init__(self):
        self.url = 'https://www.cnblogs.com/cate/csharp/#p5'
        self.checkMsg = ''
        self.fileName = path.join(d, 'image/')
        self.fileDirName = ''
        self.chrome_options = Options()
        self.chrome_options.add_argument('--headless')
        self.chrome_options.add_argument('--disable-gpu')
        self.driver = webdriver.Chrome(chrome_options=self.chrome_options)
        self.topHostPostfix = (
        '.com', '.la', '.io', '.co', '.info', '.net', '.org', '.me', '.mobi',
        '.us', '.biz', '.xxx', '.ca', '.co.jp', '.com.cn', '.net.cn',
        '.org.cn', '.mx', '.tv', '.ws', '.ag', '.com.ag', '.net.ag',
        '.org.ag', '.am', '.asia', '.at', '.be', '.com.br', '.net.br',
        '.bz', '.com.bz', '.net.bz', '.cc', '.com.co', '.net.co',
        '.nom.co', '.de', '.es', '.com.es', '.nom.es', '.org.es',
        '.eu', '.fm', '.fr', '.gs', '.in', '.co.in', '.firm.in', '.gen.in',
        '.ind.in', '.net.in', '.org.in', '.it', '.jobs', '.jp', '.ms',
        '.com.mx', '.nl', '.nu', '.co.nz', '.net.nz', '.org.nz',
        '.se', '.tc', '.tk', '.tw', '.com.tw', '.idv.tw', '.org.tw',
        '.hk', '.co.uk', '.me.uk', '.org.uk', '.vg', ".com.hk")

    def inputUrl(self):
        '''input url'''
        self.url = input('please input your target: ')
        print('[*] url: %s' % self.url)

    def check(self):
        '''check url'''
        self.checkMsg = input('Are your sure to grab this site? [Y/N/Exit] :')
        if self.checkMsg == 'Y':
            self.middle = self.url.replace('http://', '')
            self.middle = self.middle.replace('https://', '')
            self.fileDirName = path.join(d, 'image/%s' % self.middle)
            self.makeFile()
            self.parse()
        elif self.checkMsg == 'N':
            self.inputUrl()
            self.check()
        elif self.checkMsg == 'Exit':
            sys.exit()
        else:
            print('please input one of [Y/N/Exit]!!')
            self.check()

    def makeFile(self):
        '''创建文件夹函数'''
        if os.path.exists(self.fileName):
            pass
        else:
            os.makedirs(self.fileName)

        if os.path.exists(self.fileDirName):
            pass
        else:
            os.makedirs(self.fileDirName)

    def getCssImage(self,url):
        '''获取css中的image'''
        headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}
        try:
            response = requests.get(url, headers = headers, timeout=500).text
            bgCssList = re.findall("url\((.*?\))", response)
            bgCssSrc = []
            if len(bgCssList) > 0:
                for v in bgCssList:
                    v = v.replace('url(', '')
                    v = v.replace('\\', "")
                    v = v.replace(')', "")
                    print(v)
                    print('-----------------------------------')
                    bgCssSrc.append(v)
            return bgCssSrc
        except:
            print('connection timeout!!!')

    def getHostName(self, url):
        '''获取url主域名'''
        regx = r'[^\.]+(' + '|'.join([h.replace('.', r'\.') for h in self.topHostPostfix]) + ')$'
        pattern = re.compile(regx, re.IGNORECASE)
        parts = urlparse(self.url)
        host = parts.netloc
        m = pattern.search(host)
        urlm = 'http://www.' + m.group() if m else host
        return urlm

    def joinUrl(self, url):
        '''图片url处理'''
        # if url[:2] == '//':
        #     url = url.replace('//', '')
        #     url = 'http://' + url
        # elif url.startswith('/'):
        #     ## 需要处理
        #     regx = r'[^\.]+(' + '|'.join([h.replace('.', r'\.') for h in self.topHostPostfix]) + ')$'
        #     pattern = re.compile(regx, re.IGNORECASE)
        #     parts = urlparse(self.url)
        #     host = parts.netloc
        #     m = pattern.search(host)
        #     urlm = 'http://www.' + m.group() if m else host
        #     url = urlm + url
        # try:
        #     ## 处理字符串   获取 www   http   https
        #     if url[:2] == '//':
        #         url = url.replace('//', '')
        #         url = 'http://' + url
        #     elif url.startswith('/'):
        #         ## 需要处理
        #         regx = r'[^\.]+(' + '|'.join([h.replace('.', r'\.') for h in self.topHostPostfix]) + ')$'
        #         pattern = re.compile(regx, re.IGNORECASE)
        #         parts = urlparse(self.url)
        #         host = parts.netloc
        #         m = pattern.search(host)
        #         urlm = 'http://www/' + m.group() if m else host
        #         url = urlm + url
        #     else:
        #         try:
        #             url = url.split('www', 1)[1]
        #             url = u'http://www' + url
        #         except:
        #             try:
        #                 url = url.split('http', 1)[1]
        #                 url = u'http' + url
        #             except:
        #                 pass
        # except:
        #     pass
        ## ex1    '//example.png'
        ## ex2    'http://'
        if url.startswith('http'):
            return url
        else:
            return urljoin(self.url, url)

    def download(self, key, url):
        if key == 0:
            pass
        else:
            print('')
        url = self.joinUrl(url)
        try:
            imgType = os.path.split(url)[1]
            imgType = imgType.split('.',1)[1]
            imgType = imgType.split('?',1)[0]
        except:
            msg = u' Error '
            return msg
        fileName = int(time.time())
        path = self.fileDirName+ u'/'+str(fileName) + u'.' + imgType
        try:
            headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}
            try:
                response = requests.get(url, headers=headers, timeout=500).content
            except:
                msg = u' Error '
                return msg
            f = open(path, 'wb+')
            try:
                f.write(response.encode('utf-8'))
            except:
                f.write(response)
            f.close()
        except Exception as e:
            msg = u' Error '
            return msg
        return u'Success'

    def parse(self):
        '''parse html'''
        self.driver.get(self.url)
        time.sleep(3)
        html_content = self.driver.page_source
        bs = BeautifulSoup(html_content, "html.parser")
        ## 先获取所有的图片
        imgList = bs.find_all('img')
        srcList = []
        if len(imgList) > 0:
            for v in imgList:
                srcList.append(v['src'])
                print(v['src'])
                print('-----------------------------------')
            srcList = list(set(srcList))
        print('[*] Find %s image in page',len(srcList))
        ## 获取当前页面style里面的背景图
        bgStyleList = re.findall("url\((.*?\))", html_content)
        bgSrc = []
        if len(bgStyleList) > 0:
            for v in bgStyleList:
                v = v.replace('url(', '')
                v = v.replace('\\',"")
                v = v.replace(')', "")
                print(v)
                print('-----------------------------------')
                bgSrc.append(v)
        bgSrc = list(set(bgSrc))
        print('[*] Find %s image in Page style', len(bgSrc))
        ## 获取所有的背景图
        ## 获取所有的css文件
        cssList = re.findall('<link rel="stylesheet" href="(.*?)"',html_content)
        cssImageUrls = []
        if len(cssList) > 0:
            cssImageUrl = []
            for url in cssList:
                cssImageUrl += self.getCssImage(url)
            cssImageUrls = cssImageUrl
            cssImageUrls = list(set(cssImageUrls))
        print('[*] Find %s image in Page css', len(cssImageUrls))

        ## 开始获取图片https://www.cnblogs.com/shuangzikun/
        ## 开始下载标签的图片
        print('---------------------------------------------')


        if len(srcList) > 0:
            print('Start Load Image -- %s' % len(srcList))
            for percent,url in enumerate(srcList):
                percent += 1
                msg = self.download(percent, url)
                output(srcList, percent, msg ,url)


        if len(bgSrc) >0:
            print('\nStart Load Image In Style -- %s' % len(bgSrc))
            for percent, url in enumerate(bgSrc):
                percent += 1
                msg = self.download(percent, url)
                output(srcList, percent, msg, url)


        if len(cssImageUrls) > 0:
            print('\nStart Load Image In Css -- %s' % len(cssImageUrls))
            for percent, url in enumerate(cssImageUrls):
                percent += 1
                msg = self.download(percent, url)
                output(srcList, percent, msg, url)

        print('\nEnd----------------------------------Exit')



if __name__ == '__main__':
    print('''          ____  __  __  __      __       _______  _______
         /__  \\ \\_\\/_/ / /     / /____  / ___  / / ___  /
        / /_/ /  \\__/ / /___  / /__  / / /  / / / /  / /
       / ____/   / / / /___/ / /  / / / /__/ / / /  / /
      /_/       /_/ /_/___/ /_/  /_/  \\_____/ /_/  /_/ version 3.6''')
    descriptionL = ['T', 'h', 'i', 's', ' ', 'i', 's' , ' ', 'a', ' ', 's', 'p', 'i', 'd', 'e', 'r', ' ','p', 'r', 'o', 'c', 'e', 'd', 'u', 'r', 'e', ' ', '-', '-', '-',' IMGSPIDER', '\n']

    for j in range(len(descriptionL)):
        sys.stdout.write(descriptionL[j])
        sys.stdout.flush()
        time.sleep(0.1)
    urlL = ['[First Step]', ' input ', 'a', ' url ' , 'as ', 'your ', 'target ~ \n']

    for j in range(len(urlL)):
        sys.stdout.write(urlL[j])
        sys.stdout.flush()
        time.sleep(0.2)
    pathL = ['[Second Step]', ' check ', 'this ', 'url ~\n']

    for j in range(len(pathL)):
        sys.stdout.write(pathL[j])
        sys.stdout.flush()
        time.sleep(0.2)
    ## new spider
    MySpider = Spider()
    ## input url path
    MySpider.inputUrl()
    # ## checkMsg
    MySpider.check()

        try:
            self.chrome_options.add_argument(r"user-data-dir = %s" % path.join('Chrome\Application'))
            self.driver = webdriver.Chrome(path.join(d,'chromedriver.exe'),chrome_options=self.chrome_options)
        except Exception as e:
            print(e)