[toc]

MapReduce之Job工具类开发

在MapReduce程序写Mapper和Reducer的驱动程序时，有很多代码都是重复性代码，因此可以将其提取出来写成一个工具类，后面再写MapReduce程序时都会使用这个工具类。

Job工具类开发

程序代码如下：

package com.uplooking.bigdata.common.utils;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;public class MapReduceJobUtil {public static Job buildJob(Configuration conf,Class<?> jobClazz,String inputpath,Class<? extends InputFormat> inputFormat,Class<? extends Mapper> mapperClass,Class<?> mapKeyClass,Class<?> mapValueClass,Path outputpath,Class<? extends OutputFormat> outputFormat,Class<? extends Reducer> reducerClass,Class<?> outkeyClass,Class<?> outvalueClass) throws IOException {String jobName = jobClazz.getSimpleName();Job job = Job.getInstance(conf, jobName);//设置job运行的jarjob.setJarByClass(jobClazz);//设置整个程序的输入FileInputFormat.setInputPaths(job, inputpath);job.setInputFormatClass(inputFormat);//就是设置如何将输入文件解析成一行一行内容的解析类//设置mapperjob.setMapperClass(mapperClass);job.setMapOutputKeyClass(mapKeyClass);job.setMapOutputValueClass(mapValueClass);//设置整个程序的输出outputpath.getFileSystem(conf).delete(outputpath, true);//如果当前输出目录存在，删除之，以避免.FileAlreadyExistsExceptionFileOutputFormat.setOutputPath(job, outputpath);job.setOutputFormatClass(outputFormat);//设置reducer，如果有才设置，没有的话就不用设置if (null != reducerClass) {job.setReducerClass(reducerClass);job.setOutputKeyClass(outkeyClass);job.setOutputValueClass(outvalueClass);}return job;}
}