利用webmagic爬去招聘信息，并输入到Hbase数据库中

首先查看我们要爬去的网站httprclyrcnetCompanyzpaspxPage1
好了该查看我们要爬到数据是
接着是爬取信息类
下面是DaoDaoImpl类
工具类如下
整体的框架如图

1. 首先查看我们要爬去的网站http://rc.lyrc.net/Companyzp.aspx?Page=1

这是一个典型的列表页＋详情页情景，而web magic就是对这样的情况非常适合。让我们们说一下什么是web magic它是模仿python 的scrapy，特点主要有
●完全模块化的设计，强大的可扩展性。
●核心简单但是涵盖爬虫的全部流程，灵活而强大，也是学习爬虫入门的好材料。
●提供丰富的抽取页面API。
●无配置，但是可通过POJO+注解形式实现一个爬虫。
●支持多线程。
●支持分布式。
●支持爬取js动态渲染的页面。
●无框架依赖，可以灵活的嵌入到项目中去。
对我这种小白来说，是最好的入门的。
它的官方文档地址http://webmagic.io/docs/zh/

2. 好了该查看我们要爬到数据是

用一个user类进行封装

package linyirencaiwang;public class User {private String key;//keywordprivate String name;//用户名private String sex;//性别private String minzu;//民族private String location;//所在地private String identity;//身份学历private String school;//学校private String major;//专业private String work_experience;//工作经验private String hope_position;//希望求职岗位private String hope_palce;//希望工作地点private String hope_salary;//希望待遇private String work_type;//希望工作类型public String getMinzu() {return minzu;}public void setMinzu(String minzu) {this.minzu = minzu;}public String getWork_experience() {return work_experience;}public void setWork_experience(String work_experience) {this.work_experience = work_experience;}public String getHope_position() {return hope_position;}public void setHope_position(String hope_position) {this.hope_position = hope_position;}public String getHope_palce() {return hope_palce;}public void setHope_palce(String hope_palce) {this.hope_palce = hope_palce;}public String getHope_salary() {return hope_salary;}public void setHope_salary(String hope_salary) {this.hope_salary = hope_salary;}public String getWork_type() {return work_type;}public void setWork_type(String work_type) {this.work_type = work_type;}public String getKey() {return key;}public void setKey(String key) {this.key = key;}public String getName() {return name;}public void setName(String name) {this.name = name;}public String getIdentity() {return identity;}public void setIdentity(String identity) {this.identity = identity;}public String getLocation() {return location;}public void setLocation(String location) {this.location = location;}public String getSex() {return sex;}public void setSex(String sex) {this.sex = sex;}public String getSchool() {return school;}public void setSchool(String school) {this.school = school;}public String getMajor() {return major;}public void setMajor(String major) {this.major = major;}@Overridepublic String toString() {return "User [name=" + name+ ", sex=" + sex + ", minzu=" + minzu + ", location="+ location+ ", identity=" + identity + ", school=" + school + ", major=" + major + ", work_experience=" + work_experience+ ", hope_position=" +hope_position  + ", hope_palce=" + hope_palce + ", hope_salary=" +hope_salary+ ", work_type=" +work_type + "]";}
}

3. 接着是爬取信息类

这里用于了webmagic框架，只要写好对应列表页url和详细url的正则表达式，webmagic就会进行匹配，就像代码里写的如果匹配到列表页的url，就把该页的详细页的url和余下的列表页添加到队列，否则利用xpath获取详细页的信息。

package linyirencaiwang;
import java.util.ArrayList;
import java.util.List;import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
public class  Test implements PageProcessor {private LinyirencaiDao LinyirencaiDao = new LinyircDaoImpL();public static final String URL_LIST ="http://rc\\.lyrc\\.net/Companyzp\\.aspx\\?Page=[1-9]{1,3}";public static final String URL_POST="/Person_Lookzl/id-[0-9]{4}\\.html";// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等static int size=1;private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);@Overridepublic void process(Page page) {// 部分二：定义如何抽取页面信息，并保存下来List<String> urls = page.getHtml().css("div#paging").links().regex("/Companyzp\\.aspx\\?Page=").all();if(page.getUrl().regex(URL_LIST).match()){page.addTargetRequests(page.getHtml().links().regex(URL_POST).all());page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());page.addTargetRequests(urls);}else{System.out.println("第"+size+"条");size++;User user =new User();String key="0";//keywordString name =page.getHtml().xpath("//*[@width='61%']/table/tbody/tr[1]/td[2]/text()").get();//用户名String sex= page.getHtml().xpath("//*[@width='61%']/table/tbody/tr[1]/td[4]/text()").get();//性别String minzu=page.getHtml().xpath("//*[@width='61%']/table/tbody/tr[2]/td[4]/text()").get().toString();//民族String location= page.getHtml().xpath("//*[@width='61%']/table/tbody/tr[3]/td[4]/text()").get();//所在地String identity=page.getHtml().xpath("//*td[@width='283']/text()").get();//身份学历String school=page.getHtml().xpath("//*td[@width='201']/text()").get();//学校String major=page.getHtml().xpath("//*[@width='90%']/tbody/tr[2]/td[4]/text()").get();//专业String work_experience=page.getHtml().xpath("//td[@width='773']/table/tbody/tr/td/table[6]/tbody/tr[2]/td[2]/text()").get();//工作经验String hope_position=page.getHtml().xpath("//td[@width='773']/table/tbody/tr/td/table[8]/tbody/tr[5]/td[2]/text()").get();//希望求职岗位String hope_palce=page.getHtml().xpath("//td[@width='773']/table/tbody/tr/td/table[8]/tbody/tr[4]/td[2]/text()").get();//希望工作地点String hope_salary=page.getHtml().xpath("//td[@width='773']/table/tbody/tr/td/table[8]/tbody/tr[2]/td[2]/text()").get();//希望待遇String work_type=page.getHtml().xpath("//td[@width='773']/table/tbody/tr/td/table[8]/tbody/tr[1]/td[2]/text()").get();user.setHope_palce(name);user.setHope_palce(hope_palce);user.setHope_position(hope_position);user.setHope_salary(hope_salary);user.setIdentity(identity); user.setKey(key);user.setLocation(location);user.setMajor(major);user.setMinzu(minzu);user.setName(name);user.setSchool(school);user.setSex(sex);user.setWork_experience(work_experience);user.setWork_type(work_type);System.out.println(user.toString());System.out.println();LinyirencaiDao.saveUser(user);}// 部分三：从页面发现后续的url地址来抓取}@Overridepublic Site getSite() {return site;}public static void main(String args[]) {long startTime, endTime;startTime =System.currentTimeMillis();System.out.println("【爬虫开始】请耐心等待一大波数据到你碗里来...");Spider.create(new Test()).addUrl("http://rc.lyrc.net/Companyzp.aspx?Page=1")//.addPipeline(new FilePipeline("D:\\webmagic\\"))//.addPipeline(new ConsolePipeline).thread(5).run();endTime = System.currentTimeMillis();System.out.println("【爬虫结束】共抓取" + size + "篇文章，耗时约" + ((endTime - startTime) / 1000) + "秒，已保存到数据库，请查收！");}}

4. 下面是Dao、DaoImpl类

在数据库中建表和设计列簇。由于这次仅是一个小demo。所以就简单建一个person2 表，列簇是info，经过我的尝试，爬取信息的时间太长，为了运行时间简短一点，先把名字录入。

package linyirencaiwang;public interface LinyirencaiDao {public void saveUser(User user);
}

package linyirencaiwang;import java.io.IOException;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;public class LinyircDaoImpL implements LinyirencaiDao {@Overridepublic void saveUser(User user)  {// TODO Auto-generated method stubConfiguration conf = HBaseConfiguration.create();conf.set("hbase.zookeeper.quorum", "192.168.6.250,192.168.6.251,192.168.6.252");try {HBHelper helper = HBHelper.getHelper(conf);if(!helper.existsTable("person2")){helper.createTable("person2",  "info");}//  helper.dropTable("person");//  helper.createTable("person",  "info");helper.insterRow("person2", user.getName(), "info", "name", user.getName());
//           helper.insterRow("person", user.getName(), "info", "sex", user.getSex());
//           helper.insterRow("person", user.getName(), "info", "sex", user.getMinzu());
//           helper.getData("person", "row1","info");//helper.close();} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();}}}

5. 工具类如下

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;public class HBHelper {private static Connection connection = null;private static Admin admin = null;public HBHelper (Configuration conf) {try {connection = HBHelper.getConnInstance(conf);} catch (Exception e) {e.printStackTrace();}} private static synchronized Connection getConnInstance(Configuration conf)throws IOException{if(connection==null){try{connection = ConnectionFactory.createConnection(conf);admin = connection.getAdmin();}catch (IOException e) {e.printStackTrace();} System.out.println("连接数据库成功");}return connection;}public  void close(){  try {  if(null != admin)  admin.close();  if(null != connection)  connection.close();  System.out.println("关闭数据库成功");} catch (IOException e) {  e.printStackTrace();  }  }  public void createTable(String table, String... colfams)throws IOException {createTable(table, null, colfams);}public void createTable(String table, byte[][] splitKeys, String... colfams)throws IOException {HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));for (String cf : colfams) {HColumnDescriptor coldef = new HColumnDescriptor(cf);desc.addFamily(coldef);}if (splitKeys != null) {admin.createTable(desc, splitKeys);} else {admin.createTable(desc);}}/*** @Title disableTable* @Description 禁用table* @param table* @throws IOException*/public void disableTable(String table) throws IOException {
//              admin.disableTable(table);admin.disableTable(TableName.valueOf(table));}public boolean existsTable(String table)throws IOException {return admin.tableExists(TableName.valueOf(table));}/*** @Title dropTable* @Description dropTable* @param table* @throws IOException*/public void dropTable(String table) throws IOException {if (existsTable(table)) {disableTable(table);admin.deleteTable(TableName.valueOf(table));}}public void insterRow(String tableName,String rowkey,String colFamily,String col,String val) throws IOException {  Table table = connection.getTable(TableName.valueOf(tableName));  Put put = new Put(Bytes.toBytes(rowkey));  put.addColumn(Bytes.toBytes(colFamily), Bytes.toBytes(col), Bytes.toBytes(val));  table.put(put);  //批量插入  /* List<Put> putList = new ArrayList<Put>(); puts.add(put); table.put(putList);*/  table.close();  }  //格式化输出  public  void showCell(Result result){  Cell[] cells = result.rawCells(); for(Cell cell:cells){  System.out.println("RowName:"+new String(CellUtil.cloneRow(cell))+", "+"Timetamp:"+cell.getTimestamp()+", "+"column Family:"+new String(CellUtil.cloneFamily(cell))+", "+"列修饰符:"+new String(CellUtil.cloneQualifier(cell))+", "+"value:"+new String(CellUtil.cloneValue(cell))+" ");  }  }  //批量查找数据  public void scanData(String tableName/*,String startRow,String stopRow*/)throws IOException{  Table table = connection.getTable(TableName.valueOf(tableName));  Scan scan = new Scan();  //scan.setStartRow(Bytes.toBytes(startRow));  //scan.setStopRow(Bytes.toBytes(stopRow));  ResultScanner resultScanner = table.getScanner(scan);  for(Result result : resultScanner){  showCell(result);  }  table.close();  }  }

6. 整体的框架如图