2）JS动态生成HTML元素的爬取

import java.util.List;import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;import cn.edu.hfut.dmic.webcollector.crawler.DeepCrawler;
import cn.edu.hfut.dmic.webcollector.model.Links;
import cn.edu.hfut.dmic.webcollector.model.Page;/** JS爬取* Refer: http://blog.csdn.net/smilings/article/details/7395509*/
public class WebCollector3 extends DeepCrawler {public WebCollector3(String crawlPath) {super(crawlPath);// TODO Auto-generated constructor stub}@Overridepublic Links visitAndGetNextLinks(Page page) {/*HtmlUnitDriver可以抽取JS生成的数据*/
//      HtmlUnitDriver driver=PageUtils.getDriver(page,BrowserVersion.CHROME);
//      String content = PageUtils.getPhantomJSDriver(page);WebDriver driver = PageUtils.getWebDriver(page);
//        List<WebElement> divInfos=driver.findElementsByCssSelector("#feed_content");List<WebElement> divInfos=driver.findElements(By.cssSelector("#feed_content span"));for(WebElement divInfo:divInfos){System.out.println("Text是：" + divInfo.getText());}return null;}public static void main(String[] args) {WebCollector3 crawler=new WebCollector3("/home/hu/data/wb");for(int page=1;page<=5;page++)
//        crawler.addSeed("http://www.sogou.com/web?query="+URLEncoder.encode("编程")+"&page="+page);crawler.addSeed("http://cq.qq.com/baoliao/detail.htm?294064");try {crawler.start(1);} catch (Exception e) {e.printStackTrace();}}}

PageUtils.java

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import org.openqa.selenium.ie.InternetExplorerDriver;
import org.openqa.selenium.phantomjs.PhantomJSDriver;import com.gargoylesoftware.htmlunit.BrowserVersion;import cn.edu.hfut.dmic.webcollector.model.Page;public class PageUtils {public static HtmlUnitDriver getDriver(Page page) {HtmlUnitDriver driver = new HtmlUnitDriver();driver.setJavascriptEnabled(true);driver.get(page.getUrl());return driver;}public static HtmlUnitDriver getDriver(Page page, BrowserVersion browserVersion) {HtmlUnitDriver driver = new HtmlUnitDriver(browserVersion);driver.setJavascriptEnabled(true);driver.get(page.getUrl());return driver;}public static WebDriver getWebDriver(Page page) {
//      WebDriver driver = new HtmlUnitDriver(true);//     System.setProperty("webdriver.chrome.driver", "D:\\Installs\\Develop\\crawling\\chromedriver.exe");
//      WebDriver driver = new ChromeDriver();System.setProperty("phantomjs.binary.path", "D:\\Installs\\Develop\\crawling\\phantomjs-2.0.0-windows\\bin\\phantomjs.exe");WebDriver driver = new PhantomJSDriver();driver.get(page.getUrl());//       JavascriptExecutor js = (JavascriptExecutor) driver;
//      js.executeScript("function(){}");return driver;}public static String getPhantomJSDriver(Page page) {Runtime rt = Runtime.getRuntime();Process process = null;try {process = rt.exec("D:\\Installs\\Develop\\crawling\\phantomjs-2.0.0-windows\\bin\\phantomjs.exe " + "D:\\workspace\\crawlTest1\\src\\crawlTest1\\parser.js " +page.getUrl().trim());InputStream in = process.getInputStream();InputStreamReader reader = new InputStreamReader(in, "UTF-8");BufferedReader br = new BufferedReader(reader);StringBuffer sbf = new StringBuffer();String tmp = "";while((tmp = br.readLine())!=null){    sbf.append(tmp);    }return sbf.toString();} catch (IOException e) {e.printStackTrace();}return null;}
}

2.1）HtmlUnitDriver getDriver是selenium 1.x的作法，已经outdate了，现在用WebDriver getWebDriver

2.2）这里用了几种方法：HtmlUnitDriver, ChromeDriver, PhantomJSDriver, PhantomJS，参考 http://blog.csdn.net/five3/article/details/19085303，各自之间的优缺 点如下：

* 真实浏览器driver 包括 Firefox, Chrome, IE

2.3）用PhantomJSDriver的时候，遇上错 误：ClassNotFoundException: org.openqa.selenium.browserlaunchers.Proxies，原 因竟然是selenium 2.44 的bug，后来通过maven找到phantomjsdriver-1.2.1.jar 才解决了。

2.4）另外，我还试了PhantomJS 原生调用（也就是不用selenium，直接调用PhantomJS，见上面的方法），原生要调用JS，这里的parser.js代码如下：

system = require('system')
address = system.args[1];//获得命令行第二个参数 接下来会用到
//console.log('Loading a web page');
var page = require('webpage').create();
var url = address;
//console.log(url);
page.open(url, function (status) {   //Page is loaded!   if (status !== 'success') {   console.log('Unable to post!');   } else {    //此处的打印，是将结果一流的形式output到java中，java通过InputStream可以获取该输出内容console.log(page.content);   }      phantom.exit();
});

3）后话

3.1）HtmlUnitDriver + PhantomJSDriver是当前最可靠的动态抓取方案。

3.2）这过程中用到很多包、exe，遇到很多的墙~，有需要的朋友可以找我要。

Reference

http://www.ibm.com/developerworks/cn/web/1309_fengyq_seleniumvswebdriver/
http://blog.csdn.net/smilings/article/details/7395509
http://phantomjs.org/download.html
http://blog.csdn.net/five3/article/details/19085303
http://phantomjs.org/quick-start.html

... ...

动态网页爬取例子（WebCollector+selenium+phantomjs）