$ pip install selenium

from selenium import webdriverdriver = webdriver.PhantomJS() # or add to your PATH
driver.set_window_size(1024, 768) # optional
driver.get('https://google.com/')
driver.save_screenshot('screen.png') # save a screenshot to disk
sbtn = driver.find_element_by_css_selector('button.gbqfba')
sbtn.click()

driver = webdriver.PhantomJS() # or add to your PATH

driver = webdriver.PhantomJS(executable_path='/usr/local/lib/node_modules/phantomjs/lib/phantom/bin/phantomjs')

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.keys import Keysbrowser = webdriver.PhantomJS(executable_path='/home/hzh/hzh/soft/phantomjs/bin/phantomjs')
browser.set_window_size(1120, 720)
browser.get("https://baidu.com/")browser.find_element_by_xpath(".//*[@id='kw']").send_keys("hzh")
# browser.find_element_by_xpath(".//*[@id='kw']").send_keys(Keys.ENTER)
browser.find_element_by_xpath(".//*[@id='su']").click()delay = 5 # seconds
try:myElem = WebDriverWait(browser, delay).until(EC.presence_of_element_located((By.XPATH, ".//*[@id='help']/a[1]")))print("success get page")
except TimeoutException:print("Loading took too much time!")print(browser.current_url)
browser.save_screenshot('/home/hzh/screen.png')browser.quit()

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.keys import Keysfrom selenium import webdriverdef init_phantomjs_driver(*args, **kwargs):headers = { 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',"Accept-Encoding": "gzip",'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36','Connection': 'keep-alive'}for key, value in enumerate(headers):webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.{}'.format(key)] = valuewebdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.settings.userAgent'] = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36'driver =  webdriver.PhantomJS(executable_path='/home/hzh/hzh/soft/phantomjs/bin/phantomjs')driver.set_window_size(1120, 720)return driverservice_args = ['--proxy=127.0.0.1:9999','--proxy-type=http','--ignore-ssl-errors=true']
browser = init_phantomjs_driver(service_args=service_args)
browser.get("https://www.huobi.com/")# browser.find_element_by_xpath(".//*[@id='kw']").send_keys("hzh")
# browser.find_element_by_xpath(".//*[@id='kw']").send_keys(Keys.ENTER)
# browser.find_element_by_xpath(".//*[@id='su']").click()
delay = 5 # seconds
try:myElem = WebDriverWait(browser, delay).until(EC.presence_of_element_located((By.XPATH, ".//*[@id='doc_body']/div[7]/div/div[1]/div[1]")))print("success get page")
except TimeoutException:print("Loading took too much time!")print(browser.current_url)
browser.save_screenshot('/home/hzh/screen.png')browser.quit()

1,  Download geckodriver
2,  Copy geckodriver in /usr/local/bin

from selenium.webdriver.common.desired_capabilities import DesiredCapabilitiesfirefox_capabilities = DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True
firefox_capabilities['binary'] = '/home/hzh/hzh/soft/firefox'

profile = webdriver.FirefoxProfile('/home/hzh/.mozilla/firefox/f3dcxoyp.default')profile.add_extension("/home/hzh/.mozilla/firefox/f3dcxoyp.default/extensions/xpath_finder@xpath_finder.com.xpi")profile.add_extension("/home/hzh/.mozilla/firefox/f3dcxoyp.default/extensions/FireXPath@pierre.tholence.com.xpi")profile.add_extension("/home/hzh/.mozilla/firefox/f3dcxoyp.default/extensions/firefinder@robertnyman.com.xpi")driver = webdriver.Firefox(firefox_profile=profile, capabilities=firefox_capabilities)
driver = webdriver.Firefox(capabilities=firefox_capabilities)

test_link = browser.find_element_by_xpath(".//*[@id='doc_head']/div/div[3]/ul/li[1]/a")
# Save the window opener (current window, do not mistaken with tab... not the same)
main_window = browser.current_window_handle
time.sleep(2)# Open the link in a new tab by sending key strokes on the element
# Use: Keys.CONTROL + Keys.SHIFT + Keys.RETURN to open tab on top of the stack
test_link.send_keys(Keys.CONTROL + Keys.RETURN)  # 在某个连接上使用 ctrl+enter 键在新的tab中打开该链接
time.sleep(2)# Get the list of window handles
tabs = browser.window_handles
print(len(tabs))
# Use the list of window handles to switch between windows
browser.switch_to_window(tabs[1])
test_link2 = browser.find_element_by_xpath(".//*[@id='doc_body']/div[4]/div[2]/div[1]/h2")
print(test_link2.text)
time.sleep(2)# Switch back to original window
browser.switch_to_window(main_window)

browser = webdriver.Firefox()
browser.get('https://www.google.com?q=python#q=python')
first_result = ui.WebDriverWait(browser, 15).until(lambda browser: browser.find_element_by_class_name('rc'))
first_link = first_result.find_element_by_tag_name('a')# Save the window opener (current window, do not mistaken with tab... not the same)
main_window = browser.current_window_handle# Open the link in a new tab by sending key strokes on the element
# Use: Keys.CONTROL + Keys.SHIFT + Keys.RETURN to open tab on top of the stack
first_link.send_keys(Keys.CONTROL + Keys.RETURN)                # 再某个连接上使用 ctrl+enter 键在新的tab中打开该链接# Switch tab to the new tab, which we will assume is the next one on the right
browser.find_element_by_tag_name('body').send_keys(Keys.CONTROL + Keys.TAB)       # 在第一个窗口上使用 ctrl+tab 键切换到下一个tab# Put focus on current window which will, in fact, put focus on the current visible tab
browser.switch_to_window(main_window)           # 再切换回第一个tab# do whatever you have to do on this page, we will just got to sleep for now
sleep(2)# Close current tab
browser.find_element_by_tag_name('body').send_keys(Keys.CONTROL + 'w')      # 关闭这个tab# Put focus on current window which will be the window opener
browser.switch_to_window(main_window)      

curWindowHndl = browser.current_window_handle
elem.send_keys(Keys.CONTROL + Keys.ENTER) #open link in new tab keyboard shortcut
sleep(2) #wait until new tab finishes loading
browser.switch_to_window(browser.window_handles[1]) #assuming new tab is at index 1
browser.close() #closes new tab
browser.switch_to_window(curWindowHndl)

scrapy-splash简介

scrapy-splash利用Splash将javascript和Scrapy集成起来，使得Scrapy可以抓取动态网页。

Splash是一个javascript渲染服务，是实现了HTTP API的轻量级浏览器，底层基于Twisted和QT框架，Python语言编写。所以首先你得安装Splash实例

安装docker

官网建议使用docker容器安装方式Splash。那么首先你得先安装docker

参考官方安装文档，这里我选择Ubuntu 12.04 LTS版本安装

升级内核版本，docker需要3.13内核

$ sudo apt-get update
$ sudo apt-get install linux-image-generic-lts-trusty
$ sudo reboot

安装CA认证

$ sudo apt-get install apt-transport-https ca-certificates

增加新的GPGkey

$ sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D

打开/etc/apt/sources.list.d/docker.list，如果没有就创建一个，然后删除任何已存在的内容，再增加下面一句

deb https://apt.dockerproject.org/repo ubuntu-precise main 

更新APT

$ sudo apt-get update
$ sudo apt-get purge lxc-docker
$ apt-cache policy docker-engine

安装

$ sudo apt-get install docker-engine

启动docker服务

$ sudo service docker start

验证是否启动成功

$ sudo docker run hello-world

上面这条命令会下载一个测试镜像并在容器中运行它，它会打印一个消息，然后退出。

安装Splash

拉取镜像下来

$ sudo docker pull scrapinghub/splash

启动容器

$ sudo docker run -p 5023:5023 -p 8050:8050 -p 8051:8051 scrapinghub/splash

现在可以通过0.0.0.0:8050(http),8051(https),5023 (telnet)来访问Splash了。

安装scrapy-splash

使用pip安装

$ pip install scrapy-splash

配置scrapy-splash

在你的scrapy工程的配置文件settings.py中添加

SPLASH_URL = 'http://192.168.203.92:8050'

添加Splash中间件，还是在settings.py中通过DOWNLOADER_MIDDLEWARES指定，并且修改HttpCompressionMiddleware的优先级

DOWNLOADER_MIDDLEWARES = {'scrapy_splash.SplashCookiesMiddleware': 723, 'scrapy_splash.SplashMiddleware': 725, 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810, } 

默认情况下，HttpProxyMiddleware的优先级是750，要把它放在Splash中间件后面

设置Splash自己的去重过滤器

DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'

如果你使用Splash的Http缓存，那么还要指定一个自定义的缓存后台存储介质，scrapy-splash提供了一个scrapy.contrib.httpcache.FilesystemCacheStorage的子类

HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'

如果你要使用其他的缓存存储，那么需要继承这个类并且将所有的scrapy.util.request.request_fingerprint调用替换成scrapy_splash.splash_request_fingerprint

使用scrapy-splash

yield SplashRequest(url, self.parse_result, args={ # optional; parameters passed to Splash HTTP API 'wait': 0.5, # 'url' is prefilled from request url # 'http_method' is set to 'POST' for POST requests # 'body' is set to request body for POST requests }, endpoint='render.json', # optional; default is render.html splash_url='<url>', # optional; overrides SPLASH_URL slot_policy=scrapy_splash.SlotPolicy.PER_DOMAIN, # optional ) 

yield scrapy.Request(url, self.parse_result, meta={ 'splash': { 'args': { # set rendering arguments here 'html': 1, 'png': 1, # 'url' is prefilled from request url # 'http_method' is set to 'POST' for POST requests # 'body' is set to request body for POST requests }, # optional parameters 'endpoint': 'render.json', # optional; default is render.json 'splash_url': '<url>', # optional; overrides SPLASH_URL 'slot_policy': scrapy_splash.SlotPolicy.PER_DOMAIN, 'splash_headers': {}, # optional; a dict with headers sent to Splash 'dont_process_response': True, # optional, default is False 'dont_send_headers': True, # optional, default is False 'magic_response': False, # optional, default is True } }) 

function main(splash)splash:init_cookies(splash.args.cookies) -- ... your script return { cookies = splash:get_cookies(), -- ... other results, e.g. html } end 

使用实例

接下来我通过一个实际的例子来演示怎样使用，我选择爬取京东网首页的异步加载内容。

京东网打开首页的时候只会将导航菜单加载出来，其他具体首页内容都是异步加载的，下面有个”猜你喜欢”这个内容也是异步加载的， 我现在就通过爬取这个”猜你喜欢”这四个字来说明下普通的Scrapy爬取和通过使用了Splash加载异步内容的区别。

首先我们写个简单的测试Spider，不使用splash：

class TestSpider(scrapy.Spider): name = "test" allowed_domains = ["jd.com"] start_urls = [ "http://www.jd.com/" ] def parse(self, response): logging.info(u'---------我这个是简单的直接获取京东网首页测试---------') guessyou = response.xpath('//div[@id="guessyou"]/div[1]/h2/text()').extract_first() logging.info(u"find：%s" % guessyou) logging.info(u'---------------success----------------') 

然后运行结果：

2016-04-18 14:42:44 test_spider.py[line:20] INFO ---------我这个是简单的直接获取京东网首页测试---------
2016-04-18 14:42:44 test_spider.py[line:22] INFO find：None
2016-04-18 14:42:44 test_spider.py[line:23] INFO ---------------success----------------

我找不到那个”猜你喜欢”这四个字

接下来我使用splash来爬取

import scrapy
from scrapy_splash import SplashRequest class JsSpider(scrapy.Spider): name = "jd" allowed_domains = ["jd.com"] start_urls = [ "http://www.jd.com/" ] def start_requests(self): splash_args = { 'wait': 0.5, } for url in self.start_urls: yield SplashRequest(url, self.parse_result, endpoint='render.html', args=splash_args) def parse_result(self, response): logging.info(u'----------使用splash爬取京东网首页异步加载内容-----------') guessyou = response.xpath('//div[@id="guessyou"]/div[1]/h2/text()').extract_first() logging.info(u"find：%s" % guessyou) logging.info(u'---------------success----------------') 

运行结果：

2016-04-18 14:42:51 js_spider.py[line:36] INFO ----------使用splash爬取京东网首页异步加载内容-----------
2016-04-18 14:42:51 js_spider.py[line:38] INFO find：猜你喜欢
2016-04-18 14:42:51 js_spider.py[line:39] INFO ---------------success----------------

可以看出结果里面已经找到了这个”猜你喜欢”，说明异步加载内容爬取成功！