hadoop 文本统计一个字符的个数_使用hadoop统计多个文本中每个单词数目
程序源码
1 importjava.io.IOException;2 importjava.util.StringTokenizer;3 importorg.apache.hadoop.conf.Configuration;4 importorg.apache.hadoop.fs.Path;5 importorg.apache.hadoop.io.IntWritable;6 importorg.apache.hadoop.io.LongWritable;7 importorg.apache.hadoop.io.Text;8 importorg.apache.hadoop.mapreduce.Job;9 importorg.apache.hadoop.mapreduce.Mapper;10 importorg.apache.hadoop.mapreduce.Reducer;11 importorg.apache.hadoop.mapreduce.lib.input.FileInputFormat;12 importorg.apache.hadoop.mapreduce.lib.input.TextInputFormat;13 importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;14 importorg.apache.hadoop.mapreduce.lib.output.TextOutputFormat;15
16 public classWordCount {17 public static class WordCountMap extends
18 Mapper{19 private final IntWritable one = new IntWritable(1);//输出的值 120 private Text word = newText();//输出的键 单词21
22 public voidmap(LongWritable key, Text value, Context context)23 throwsIOException, InterruptedException {//处理经过 TextInputFormat 产生的 ,然后产生 24 String line =value.toString();//读取文本中25 StringTokenizer token = newStringTokenizer(line);//按照空格对单词进行切割26 while(token.hasMoreTokens()) {27 word.set(token.nextToken());//读取到的单词作为键值28 context.write(word, one);//以 单词,1的中间形式交给reduce处理29 }30 }31 }32
33 public static class WordCountReduce extends
34 Reducer{35 public void reduce(Text key, Iterablevalues,36 Context context) throwsIOException, InterruptedException {37 int sum = 0;38 for(IntWritable val : values) {39 sum +=val.get();40 }41 context.write(key, newIntWritable(sum));42 }43 }44
45 public static void main(String[] args) throwsException {46 Configuration conf = newConfiguration();47 Job job = newJob(conf);48 job.setJarByClass(WordCount.class);49 job.setJobName("wordcount");50 job.setOutputKeyClass(Text.class);51 job.setOutputValueClass(IntWritable.class);52 job.setMapperClass(WordCountMap.class);53 job.setReducerClass(WordCountReduce.class);54 job.setInputFormatClass(TextInputFormat.class);//生成可供Map处理的键值对55 job.setOutputFormatClass(TextOutputFormat.class);56 FileInputFormat.addInputPath(job, new Path(args[0]));57 FileOutputFormat.setOutputPath(job, new Path(args[1]));58 job.waitForCompletion(true);59 }60 }
1 编译源码
javac -classpath /opt/hadoop-1.2.1/hadoop-core-1.2.1.jar:/opt/hadoop-1.2.1/lib/commons-cli-1.2.jar -d ./word_count_class/ WordCount.java
将源码编译成class文件并放在当前文件夹下的word_count_class目录,当然,首先需要创建该目录
2 将源码打成jar包
进入源码目录
jar -cvf wordcount.jar *
3 上传输入文件
先在hadoop中为本次任务创建一个输入文件存放目录
hadoop fs -mkdir input_wordcount
将input目录下的所有文本文件上传到hadoop中的input_wordcount目录下
hadoop fs -put input/* input_wordcount/
注意:不能在运行前穿创建输出文件夹
4 上传jar并执行
hadoop jar word_count_class/wordcount.jar input_wordcount output_wordcount
5 查看计算结果
程序输出目录
hadoop fs -ls output_wordcount
程序输出内容
hadoop fs -cat output_wordcount/part-r-00000
版本二:自己实际操作中的程序
Map程序
1 packagecom.zln.chapter03;2
3 importorg.apache.hadoop.io.IntWritable;4 importorg.apache.hadoop.io.LongWritable;5 importorg.apache.hadoop.io.Text;6 importorg.apache.hadoop.mapred.MapReduceBase;7 importorg.apache.hadoop.mapred.Mapper;8 importorg.apache.hadoop.mapred.OutputCollector;9 importorg.apache.hadoop.mapred.Reporter;10
11 importjava.io.IOException;12 importjava.util.StringTokenizer;13
14 /**
15 * Created by sherry on 15-7-12.16 */
17 public class WordCountMap extends MapReduceBase implements Mapper{18 private final static IntWritable one = new IntWritable(1);//每个单词 +1
19 private Text word = newText();20
21 @Override22 public void map(LongWritable longWritable, Text text, OutputCollector outputCollector, Reporter reporter) throwsIOException {23 String line =text.toString();24 StringTokenizer tokenizer = new StringTokenizer(line);//分割出单词
25 while(tokenizer.hasMoreTokens()){26 word.set(tokenizer.nextToken());27 outputCollector.collect(word,one);28 }29 }30 }
Reduce程序
1 packagecom.zln.chapter03;2
3 importorg.apache.hadoop.io.IntWritable;4 importorg.apache.hadoop.io.Text;5 importorg.apache.hadoop.mapred.MapReduceBase;6 importorg.apache.hadoop.mapred.OutputCollector;7 importorg.apache.hadoop.mapred.Reducer;8 importorg.apache.hadoop.mapred.Reporter;9
10 importjava.io.IOException;11 importjava.util.Iterator;12
13 /**
14 * Created by sherry on 15-7-12.15 */
16 public class WordCountReduce extends MapReduceBase implements Reducer{17 @Override18 public void reduce(Text text, Iterator iterator, OutputCollector outputCollector, Reporter reporter) throwsIOException {19 int sum = 0;20 while(iterator.hasNext()){21 sum +=iterator.next().get();22 }23 outputCollector.collect(text,newIntWritable(sum));24 }25 }
主函数
1 packagecom.zln.chapter03;2
3 importorg.apache.hadoop.fs.Path;4 importorg.apache.hadoop.io.IntWritable;5 importorg.apache.hadoop.io.Text;6 import org.apache.hadoop.mapred.*;7
8 importjava.io.IOException;9
10
11 /**
12 * Created by sherry on 15-7-12.13 */
14 public classWordCount {15 public static void main(String[] args) throwsIOException {16 JobConf conf = new JobConf(WordCount.class);17 conf.setJobName("wordCount");18
19 //设置输出格式
20 conf.setOutputKeyClass(Text.class);21 conf.setOutputValueClass(IntWritable.class);22
23 //设置MapReduce类
24 conf.setMapperClass(WordCountMap.class);25 conf.setReducerClass(WordCountReduce.class);26
27 //设置处理输入类
28 conf.setInputFormat(TextInputFormat.class);29 //设置处理输出类
30 conf.setOutputFormat(TextOutputFormat.class);31
32 FileInputFormat.setInputPaths(conf, new Path(args[0]));33 FileOutputFormat.setOutputPath(conf, new Path(args[1]));34
35 JobClient.runJob(conf);36 }37 }
准备输入文件
file1
Hello Word By Word
Hello Word By zln
file2
Hello Hadoop
Hello GoodBye
放在同一个目录下:/home/sherry/IdeaProjects/Hadoop/WordCount/输入文件准备
编译class打成一个jar包
我使用IDEA进行编译。注意不要忘记指定main函数
上传输入文件
root@sherry:/opt/hadoop-1.2.1# hadoop fs -mkdir /user/root/zln/WordCount/InputFiles
root@sherry:/opt/hadoop-1.2.1# hadoop fs -put /home/sherry/IdeaProjects/Hadoop/WordCount/输入文件准备/*/user/root/zln/WordCount/InputFiles
上传jar并执行
root@sherry:/opt/hadoop-1.2.1# hadoop jar /home/sherry/IdeaProjects/Hadoop/out/artifacts/WordCount_jar/WordCount.jar /user/root/zln/WordCount/InputFiles /user/root/zln/WordCount/OutputFiles
查看执行结果
root@sherry:/opt/hadoop-1.2.1# hadoop fs -ls /user/root/zln/WordCount/OutputFiles
root@sherry:/opt/hadoop-1.2.1# hadoop fs -text /user/root/zln/WordCount/OutputFiles/part-00000
版本三:使用新版本的API对Map Reduce main函数进行重写
Map
1 packagecom.zln.chapter03;2
3 importorg.apache.hadoop.io.IntWritable;4 importorg.apache.hadoop.io.LongWritable;5 importorg.apache.hadoop.io.Text;6 importorg.apache.hadoop.mapreduce.Mapper;7
8 importjava.io.IOException;9 importjava.util.StringTokenizer;10
11 /**
12 * Created by sherry on 15-7-12.13 */
14 public class WordCountMap extends Mapper{15 private final static IntWritable one = new IntWritable(1);//每个单词 +1
16 private Text word = newText();17
18
19 @Override20 protected void map(LongWritable key, Text value, Context context) throwsIOException, InterruptedException {21 String line =value.toString();22 StringTokenizer tokenizer = new StringTokenizer(line);//分割出单词
23 while(tokenizer.hasMoreTokens()){24 word.set(tokenizer.nextToken());25 context.write(word,one);26 }27 }28
29 }
Reduce
1 packagecom.zln.chapter03;2
3 importorg.apache.hadoop.io.IntWritable;4 importorg.apache.hadoop.io.Text;5 importorg.apache.hadoop.mapreduce.Reducer;6
7 importjava.io.IOException;8
9 /**
10 * Created by sherry on 15-7-12.11 */
12 public class WordCountReduce extends Reducer{13
14 @Override15 protected void reduce(Text key, Iterable values, Context context) throwsIOException, InterruptedException {16 int sum = 0;17 for(IntWritable intWritable:values){18 sum +=intWritable.get();19 }20 context.write(key,newIntWritable(sum));21 }22 }
Main
1 packagecom.zln.chapter03;2
3
4 importorg.apache.hadoop.conf.Configured;5 importorg.apache.hadoop.fs.Path;6 importorg.apache.hadoop.io.IntWritable;7 importorg.apache.hadoop.io.Text;8 importorg.apache.hadoop.mapreduce.Job;9 importorg.apache.hadoop.mapreduce.lib.input.FileInputFormat;10 importorg.apache.hadoop.mapreduce.lib.input.TextInputFormat;11 importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;12 importorg.apache.hadoop.mapreduce.lib.output.TextOutputFormat;13 importorg.apache.hadoop.util.Tool;14 importorg.apache.hadoop.util.ToolRunner;15
16
17
18 /**
19 * Created by sherry on 15-7-12.20 */
21 public class WordCount extends Configured implementsTool{22
23 public int run(String[] args) throwsException {24 Job job = newJob(getConf());25 job.setJarByClass(WordCount.class);26 job.setJobName("WordCount");27
28
29 job.setOutputKeyClass(Text.class);30 job.setOutputValueClass(IntWritable.class);31
32 job.setMapperClass(WordCountMap.class);33 job.setReducerClass(WordCountReduce.class);34
35 job.setInputFormatClass(TextInputFormat.class);36 job.setOutputFormatClass(TextOutputFormat.class);37
38 FileInputFormat.setInputPaths(job,new Path(args[0]));39 FileOutputFormat.setOutputPath(job,new Path(args[1]));40
41 boolean success = job.waitForCompletion(true);42 return success?0:1;43 }44
45 public static void main(String[] args) throwsException {46 int ret = ToolRunner.run(newWordCount(),args);47 System.exit(ret);48 }49 }
hadoop 文本统计一个字符的个数_使用hadoop统计多个文本中每个单词数目相关推荐
- python中怎么统计英文字符的个数_使用python统计英文文档中的字母占比和词频
本文主要介绍如何使用python软件进行统计文档中各字母出现概率和数量最多单词.程序中用到的部分库文件需自行下载,可参考网址:https://www.lfd.uci.edu/~gohlke/pytho ...
- python统计中文字符的个数_python统计不同字符的个数_后端开发
c++怎么将字符串转数字_后端开发 在"c++"中可以使用"atoi"库函数将字符串转成数字,其语法是"int atoi(const char *np ...
- hadoop 文本统计一个字符的个数_hadoop统计单词个数 - 卡饭网
hadoop入门之统计单词在文件中出现的个数示例 hadoop入门之统计单词在文件中出现的个数示例 Linux环境:CentOs6.4 Hadoop版本:hadoop-0.20.2 内容:统计hado ...
- python中怎么统计英文字符的个数_【Python练习1】统计一串字符中英文字母、空格、数字和其他字符的个数...
练习思路: 1.输入一串字符 2.筛选出字符中的英文字母并统计 3.筛选出字符中的空格并统计 4.筛选出字符中的数字并统计 5.筛选出字符中的其他字符并统计 代码实现: def msg(s): abc ...
- python中怎么统计英文字符的个数_如何统计文本中的中英文字符数?Python帮你解决...
本文的文字及图片来源于网络,仅供学习.交流使用,不具有任何商业用途,版权归原作者所有,如有问题请及时联系我们以作处理 以下文章来源于小蚊子数据分析 ,作者小蚊子数据分析 代码 1import stri ...
- Java 从键盘输入一个字符串,统计某个字符的个数
从键盘输入一个字符串,统计某个字符的个数 例如,从键盘输入"I am a student",统计a出现的次数 字符串转字符数组 字符串.toCharArray(); Scanner ...
- (篇九)C语言统计某个字母的个数、统计各种字符的个数、统计单词的个数
文章目录 一.统计某个字母的个数 二.统计各种字符的个数 三.统计单词的个数 本篇文章主要介绍在C语言中统计某个字母的个数.统计各种字符的个数和统计单词的个数:总之就是计数,-由于C语言中没有直接统计 ...
- python统计不同字符的个数
[开发该软件的操作系统]:windows10 [软件开发环境/开发工具]:PyCharm [编程语言]:Python [开发目的]:这是老师布置的作业啦~ 供初学者参考学习 [开发者]:江西农业大学2 ...
- python输入一个字符串、计算其中小写字符的个数_利用键盘录入,输入一个字符串,统计该字符串中各个字符的数量,并输出(c/c++实现)...
title: 输入一串字符串统计字符个数 localimage: image1 urlname: staticstringnum categories: summarytags: [writen, e ...
最新文章
- Linux系统编程——线程私有数据
- iOS - appledoc 文档生成
- Entity相互关系
- 每天一道LeetCode-----将单词数组分成多行,每行长度相同,单词之间用空格分隔,要求空格尽量均匀分布
- android简单服务器的搭建
- 期货与期权(part8)--市场报价
- SushiSwap MISO 遭软件供应链攻击,价值300万美元的以太坊被盗
- 理解BERT每一层都学到了什么
- android修改对话框大小设置,android – 如何更改对话框首选项消息的文本大小?...
- CenterNet :Objects as Points/CenterTrack:Tracking Objects as Points
- 破解TexturePacker加密资源
- python int转换为byte_Python int与byte类型相互转化
- 使用postman发送post请求下载文件
- 高性能计算专业应用软件大观
- 2.K8S部署-------- 制作CA证书
- 那些警示良言——老百姓也是圣贤
- 1D卷积网络HAR(人体活动识别)实践
- asterisk实现webrtc拨打电话
- 《乔布斯传》经典摘录(七)
- execlp(ls,flw,-?,(char *)0) 为什么少了最后的一个参数就不行?