1、MR设计和开发
   1)设计：
     输入：用户名 | 数字ip | 时间戳 |  url
     MR处理：正则表达式匹配url，满足则解析url并转换ip和时间戳，
     输出：用户名|点分制IP|日期时间|URL域名|URL参数；
   2)开发，见源码；
     测试文件：/tmp/fjs/in/testdata.txt
     输出文件：/tmp/fjs/out/part-r-00000.bz2
     注意：开发中务必避免陷入两个误区，一个是本地文件处理和hdfs文件处理不同； 另一个是Map和Reduce函数是在不同节点上执行，参数要能共享；
           
2、Hive表：外部表tmp_fjs_adslurl，location=/tmp/fjs/out/
   1)建表脚本：hive>
     CREATE EXTERNAL TABLE `tmp_fjs_adslurl`(`account` string,`ip` string,`time_stamp` string,`domain` string,`para` string) 
     ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' 
     STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' 
     OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
     LOCATION 'hdfs://nameservice-ha/tmp/fjs/out';
   2)查询脚本：hive>select count(*) from tmp_fjs_adslurl; 

3、代码：
    1）MapReduce代码
    其中URLLocation类解析URL提取域名和参数串采用字符串截取，可参考网上通用方法，如http://volunteer521.iteye.com/blog/1685942，这里是直接用了别人代码，未经授权不便发布。

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;public class AdslUrl {public static class UrlMapper extends Mapper<Object, Text, Text, Text>{private Text oKey = new Text();private Text oValue=new Text();Util util=new Util();public void map(Object key, Text value, Context context)throws IOException, InterruptedException {String[] iValue=value.toString().split("\\|");//获取行，并按照|分隔符提取oKey.set(iValue[0].trim());//用户名作为输出keyString ip=util.Long2IP(iValue[1].trim());//数字ip转化为点分制String dt=util.timestamp2date(iValue[2].trim());//时间戳转日期时间//解析url，提取域名和参数及其值if(util.regular(iValue[3])){//url正则匹配URLocation parser = new URLocation(" "+iValue[3].trim());//头部加空格String domain=parser.getPath();//域名String para=parser.getQuery();//参数串，&分割oValue.set(ip+"|"+dt+"|"+domain+"|"+para);context.write(oKey, oValue);}}  }public static class UrlReducer extends Reducer<Text,Text,Text,Text> {private Text oKey = new Text();public void reduce(Text key, Iterable<Text> values,Context context) throws IOException, InterruptedException {for (Text val : values) {String str=key.toString()+"|"+val.toString();oKey.set(str);context.write(oKey, new Text(""));}}}public static void main(String[] args) throws Exception {Configuration conf = new Configuration();String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();if (otherArgs.length != 2) {System.err.println("Usage: AdslUrl <in> <out>");System.exit(2);}Job job = new Job(conf, "parse url");job.setJarByClass(AdslUrl.class);job.setNumReduceTasks(1);//设置reduce输入文件一个，方便查看结果job.setMapperClass(UrlMapper.class);job.setCombinerClass(UrlReducer.class);job.setReducerClass(UrlReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(Text.class);FileInputFormat.addInputPath(job, new Path(otherArgs[0]));FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));System.exit(job.waitForCompletion(true) ? 0 : 1);}
}    

2）Util类处理字段

import java.io.BufferedReader;
import java.io.FileReader;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;public class Util {//数字型IP转为点分十进制IPpublic String Long2IP(String _ipLong){StringBuffer ipInfo = new StringBuffer(); long mask[] = {0x000000FF,0x0000FF00,0x00FF0000,0xFF000000}; long num = 0; try{Long ipLong = Long.parseLong(_ipLong);ipInfo.setLength(0);for(int i=0;i<4;i++){ num = (ipLong & mask[i])>>(i*8); if(i>0) ipInfo.insert(0,".");   ipInfo.insert(0,Long.toString(num,10)); }return ipInfo.toString(); } catch (Exception ex){return "";}}//时间戳转日期时间public String timestamp2date(String _timeStamp){String dateFormat = "yyyyMMddHHmmss";SimpleDateFormat fm = new SimpleDateFormat(dateFormat);;if (_timeStamp.equals("")){return "";}try{long timeStamp = Long.parseLong(_timeStamp);String dt = fm.format(new Date(timeStamp*1000));return dt;} catch (Exception ex){return "";}}//正则过滤域名和参数public boolean regular(String url){//正则表达式String regEx="baidu|sina|qq";Pattern pattern = Pattern.compile(regEx,Pattern.CASE_INSENSITIVE);Matcher matcher = pattern.matcher(url);if(matcher.find()) return true;return false;}
}