/*** @date 2016年8月20日 下午6:13:24

*@version*@sinceJDK 1.8*/

public class ZiroomCrawler extendsWebCrawler {/**爬取匹配原则*/

private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g|ico"

+ "|png|tiff?|mid|mp2|mp3|mp4" + "|wav|avi|mov|mpeg|ram|m4v|pdf" + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");/**爬取数据保存文件路径*/

private final static String DATA_PATH = "data/crawl/ziroom.csv";/**爬取link文件路径*/

private final static String LINK_PATH = "data/crawl/link.csv";//private static final Logger logger =//LoggerFactory.getLogger(ZiroomCrawler.class);

private final static String URL_PREFIX = "http://sh.ziroom.com/z/nl/";private finalFile fLinks;private finalFile fDatas;privateCsvWriter csvLinks;privateCsvWriter csvDatas;/*** You should implement this function to specify whether the given url

* should be crawled or not (based on your crawling logic).*/ZiroomCrawlStat myCrawlStat;public ZiroomCrawler() throwsIOException {

myCrawlStat= newZiroomCrawlStat();

fLinks= newFile(DATA_PATH);

fDatas= newFile(LINK_PATH);if(fLinks.isFile()) {

fLinks.delete();

}if(fDatas.isFile()) {

fDatas.delete();

}

csvDatas= new CsvWriter(new FileWriter(fDatas, true), ',');

csvDatas.write("请求路径");

csvDatas.endRecord();

csvDatas.close();

csvLinks= new CsvWriter(new FileWriter(fLinks, true), ',');

csvLinks.write("图片");

csvLinks.write("价格");

csvLinks.write("地址");

csvLinks.write("说明");

csvLinks.endRecord();

csvLinks.close();

}public voiddumpMyData() {final int id =getMyId();//You can configure the log to output to file

logger.info("Crawler {} > Processed Pages: {}", id, myCrawlStat.getTotalProcessedPages());

logger.info("Crawler {} > Total Links Found: {}", id, myCrawlStat.getTotalLinks());

logger.info("Crawler {} > Total Text Size: {}", id, myCrawlStat.getTotalTextSize());

}

@OverridepublicObject getMyLocalData() {returnmyCrawlStat;

}

@Overridepublic voidonBeforeExit() {

dumpMyData();

}/** 这个方法决定了要抓取的URL及其内容，例子中只允许抓取“http://sh.ziroom.com/z/nl/”这个域的页面,

* 不允许.css、.js和多媒体等文件

*

* @see edu.uci.ics.crawler4j.crawler.WebCrawler#shouldVisit(edu.uci.ics.

* crawler4j.crawler.Page, edu.uci.ics.crawler4j.url.WebURL)*/@Overridepublic booleanshouldVisit(Page referringPage, WebURL url) {final String href =url.getURL().toLowerCase();if (FILTERS.matcher(href).matches() || !href.startsWith(URL_PREFIX)) {return false;

}return true;

}/** 当URL下载完成会调用这个方法。你可以轻松获取下载页面的url, 文本, 链接, html,和唯一id等内容。

*

* @see

* edu.uci.ics.crawler4j.crawler.WebCrawler#visit(edu.uci.ics.crawler4j.

* crawler.Page)*/@Overridepublic voidvisit(Page page) {final String url =page.getWebURL().getURL();

logger.info("爬取路径：" +url);

myCrawlStat.incProcessedPages();if (page.getParseData() instanceofHtmlParseData) {final HtmlParseData htmlParseData =(HtmlParseData) page.getParseData();final Set links =htmlParseData.getOutgoingUrls();try{

linkToCsv(links);

}catch (finalIOException e2) {//TODO Auto-generated catch block

e2.printStackTrace();

}

myCrawlStat.incTotalLinks(links.size());try{

myCrawlStat.incTotalTextSize(htmlParseData.getText().getBytes("UTF-8").length);

}catch (finalUnsupportedEncodingException e1) {//TODO Auto-generated catch block

e1.printStackTrace();

}final String html =htmlParseData.getHtml();final Document doc =Jsoup.parse(html);final Elements contents = doc.select("li[class=clearfix]");for (finalElement c : contents) {//图片

final String img = c.select(".img img").first().attr("src");

logger.debug("图片：" +img);//地址

final Element txt = c.select("div[class=txt]").first();final String arr1 = txt.select("h3 a").first().text();final String arr2 = txt.select("h4 a").first().text();final String arr3 = txt.select("div[class=detail]").first().text();final String arr = arr1.concat(arr1 + ",").concat(arr2 + ",").concat(arr3);

logger.debug("地址：" +arr);//说明

final String rank = txt.select("p").first().text();

logger.debug("说明：" +rank);//价格

final String pirce = c.select("p[class=price]").first().text();try{

csvLinks= new CsvWriter(new FileWriter(fLinks, true), ',');

csvLinks.write(img);

csvLinks.write(pirce);

csvLinks.write(arr);

csvLinks.write(rank);

csvLinks.endRecord();

csvLinks.flush();

csvLinks.close();

}catch (finalIOException e) {

e.printStackTrace();

}

}

}

}private void linkToCsv(Set links) throwsIOException {

csvDatas= new CsvWriter(new FileWriter(fDatas, true), ',');for (finalWebURL webURL : links) {

csvDatas.write(webURL.getURL());

}

csvDatas.flush();

csvDatas.endRecord();

csvDatas.close();

}

}