对Mapreduce代码进行单元测试

hadoop自带一个wordcount的示例代码，用于计算单词个数。我将其单独移出来，测试成功。源码如下：

package org.apache.hadoop.examples;

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.util.GenericOptionsParser;

public class WordCount {

public static class TokenizerMapper

extends Mapper{

private final static IntWritable one = new IntWritable(1);

private Text word = new Text();

public void map(Object key, Text value, Context context

) throws IOException, InterruptedException {

StringTokenizer itr = new StringTokenizer(value.toString());

while (itr.hasMoreTokens()) {

word = new Text(itr.nextToken()); //to unitest,should be new Text word.set(itr.nextToken())

context.write(word, new IntWritable(1));

}

public static class IntSumReducer

extends Reducer {

private IntWritable result = new IntWritable();

public void reduce(Text key, Iterable values,

Context context

) throws IOException, InterruptedException {

int sum = 0;

for (IntWritable val : values) {

sum += val.get();

}

result.set(sum);

context.write(key, result);

}

public static void main(String[] args) throws Exception {

Configuration conf = new Configuration();

String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

if (otherArgs.length != 2) {

System.err.println("Usage: wordcount ");

System.exit(2);

}

Job job = new Job(conf, "word count");

job.setJarByClass(WordCount.class);

job.setMapperClass(TokenizerMapper.class);

job.setCombinerClass(IntSumReducer.class);

job.setReducerClass(IntSumReducer.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

FileInputFormat.addInputPath(job, new Path(otherArgs[0]));

FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

System.exit(job.waitForCompletion(true) ? 0 : 1);

}

现在我想对其进行单元测试。一种方式，是job执行完了后，读取输出目录中的文件，确认计数是否正确。但这样的情况如果失败，也不知道是哪里失败。我们需要对map和reduce单独进行测试。

tomwhite的书《hadoop权威指南》有提到如何用Mockito进行单元测试，我们依照原书对温度的单元测试来对wordcount进行单元测试。(原书第二版的示例已经过时，可以参考英文版第三版或我的程序)。

package org.apache.hadoop.examples;

/* author zhouhh

* date:2012.8.7

import static org.mockito.Mockito.*;

import java.io.IOException;

import java.util.ArrayList;

import java.util.List;

import org.apache.hadoop.io.*;

import org.junit.*;

public class WordCountTest {

@Test

public void testWordCountMap() throws IOException, InterruptedException

{

WordCount w = new WordCount();

WordCount.TokenizerMapper mapper = new WordCount.TokenizerMapper();

Text value = new Text("a b c b a a");

@SuppressWarnings("unchecked")

WordCount.TokenizerMapper.Context context = mock(WordCount.TokenizerMapper.Context.class);

mapper.map(null, value, context);

verify(context,times(3)).write(new Text("a"), new IntWritable(1));

verify(context).write(new Text("c"), new IntWritable(1));

//verify(context).write(new Text("cc"), new IntWritable(1));

}

@Test

public void testWordCountReduce() throws IOException, InterruptedException

{

WordCount.IntSumReducer reducer = new WordCount.IntSumReducer();

WordCount.IntSumReducer.Context context = mock(WordCount.IntSumReducer.Context.class);

Text key = new Text("a");

List values = new ArrayList();

values.add(new IntWritable(1));

reducer.reduce(key, values, context);

verify(context).write(new Text("a"), new IntWritable(2));

}

public static void main(String[] args) {

//try {

//WordCountTest t = new WordCountTest();

////t.testWordCountMap();

//t.testWordCountReduce();

//} catch (IOException e) {

//// TODO Auto-generated catch block

//e.printStackTrace();

//} catch (InterruptedException e) {

//// TODO Auto-generated catch block

//e.printStackTrace();

//}

}

verify(context)只检查一次的写，如果多次写，需用verify(contex,times(n))检查,否则会失败。

执行时在测试文件上点run as JUnit Test，会得到测试结果是否通过。

本示例程序在hadoop1.0.3环境中测试通过。Mockito也在hadoop的lib中自带，打包在mockito-all-1.8.5.jar

最新内容请见作者的GitHub页：http://qaseven.github.io/

对Mapreduce代码进行单元测试相关推荐

对编写的代码进行单元测试_编写数据访问代码测试–单元测试是浪费
对编写的代码进行单元测试几年前,我是为我的数据访问代码编写单元测试的那些开发人员之一. 我正在孤立地测试所有内容,我对自己感到非常满意. 老实说,我认为自己做得很好. 哦,男孩,我错了! 这篇博客文 ...
编写数据访问代码测试–单元测试是浪费
几年前,我是为我的数据访问代码编写单元测试的那些开发人员之一. 我正在孤立地测试所有内容,我对自己感到非常满意. 老实说,我认为自己做得很好. 哦,男孩,我错了! 这篇博客文章描述了为什么我们不应该为 ...
javascript 代码_如何开始对JavaScript代码进行单元测试
javascript 代码 We all know we should write unit tests. But, it's hard to know where to start and how ...
mapreduce代码示例_MapReduce算法示例
mapreduce代码示例 Welcome to MapReduce algorithm example. Before writing MapReduce programs in CloudEra ...
在Hadoop中用Python实现MapReduce代码
在Hadoop中用Python实现MapReduce代码利用MRjob编写MapReduce代码 1.1.1 安装Mrjob 1.1.2 mrjob实现词频统计 1.1.3 运行mrjob 运行MR ...
使用MRUnit，Mockito和PowerMock进行Hadoop MapReduce作业的单元测试
0.preliminary 环境搭建 Setup development environment Download the latest version of MRUnit jar from Apac ...
使用FakeAsync对Angular异步代码进行单元测试
The problem with async is that we still have to introduce real waiting in our tests, and this can ma ...
python可以测试java的代码吗_使用python做你自己的自动化测试--对Java代码做单元测试（2）-导入第三方jar包裹...
使用Jython对Java做单元测试,当然,为了测试开发java代码.这涉及到引入第三包的问题,如何导入第三方的包? 您可以使用http://blog.csdn.net/powerccna/artic ...
提高代码质量——使用Jest和Sinon给已有的代码添加单元测试
概述在日常的功能开发中,我们的代码测试都依赖于自己或者QA进行测试.这些操作不仅费时费力,而且还依赖开发者自身的驱动.在开发一些第三方依赖的库时,我们也没有办法给第三方提供完整的代码质量报告. 现在 ...

对Mapreduce代码进行单元测试

对Mapreduce代码进行单元测试相关推荐

最新文章

热门文章