简易下载器的实现

支持代理、失败重试、确保包含指定ID元素(可根据需求自定义修改)

# coding: utf-8

from Utils import logging

from bs4 import BeautifulSoup as bs

from selenium import webdriver

from selenium.webdriver.common.by import By

from selenium.webdriver.common.proxy import ProxyType

from selenium.webdriver.support import expected_conditions as EC

from selenium.webdriver.support.ui import WebDriverWait

class HtmlDownloader:

def __init__(self):

self.driver = webdriver.PhantomJS()

def setProxy(self, proxyStr):

# 利用DesiredCapabilities(代理设置)参数值，重新打开一个sessionId

proxy=webdriver.Proxy()

proxy.proxy_type=ProxyType.MANUAL

proxy.http_proxy=proxyStr

# 将代理设置添加到webdriver.DesiredCapabilities.PHANTOMJS中

proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)

self.driver.start_session(webdriver.DesiredCapabilities.PHANTOMJS)

def rmProxy(self):

# 还原为系统代理

proxy=webdriver.Proxy()

proxy.proxy_type=ProxyType.DIRECT

proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)

browser.start_session(webdriver.DesiredCapabilities.PHANTOMJS)

def download(self, returnType, url, ensureId, proxyStr = None):

if proxyStr:

self.setProxy(proxyStr)

else:

self.rmProxy()

self.driver.get(url)

# special for xxx.com

# your code here

# ensure for some element

try:

WebDriverWait(self.driver, 30).until(EC.presence_of_element_located((By.ID, ensureId)))

if returnType == "html":

downloadResult = self.driver.page_source

elif returnType == "bs":

downloadResult = bs(self.driver.page_source, 'lxml')

logging("i", "download %s bytes" % len(self.driver.page_source))

return downloadResult

except Exception,e:

logging("e", str(e))

finally:

self.driver.close()

def safeDownload(self, returnType, url, ensureId, proxyStr = None):

downloadResult = None

failTimes = 0

while not downloadResult:

downloadResult = self.download(returnType, url, ensureId, proxyStr)

if not downloadResult:

failTimes += 1

if failTimes == 5:

logging("w", "failed %s times, will abort" % failTimes)

break

logging("w", "failed %s times, will retry" % failTimes)

return downloadResult

元素不可见导致不能操作的错误

# ElementNotVisibleException: Message: {"errorMessage":"Element is not currently visible and may not be manipulated"

# Screenshot: available via screen

首先尝试设定窗口大小

self.driver.set_window_size(1024, 768)

不行的话再尝试滚动页面，如滚动到底部：

driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")