Hadoop入门（十三）远程提交wordCout程序到hadoop集群

一、项目结构

用到的文件有WordCount.java、core-site.xml、mapreduce-site.xml、yarn-site.xml、log4j.properties、pom.xml

二、项目源码

（1）WordCount.java

package com.mk.mapreduce;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;
import java.util.StringTokenizer;public class WordCount {public static class MyMapper extends Mapper<LongWritable, Text, Text, IntWritable> {@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {StringTokenizer tokenizer = new StringTokenizer(value.toString());while (tokenizer.hasMoreTokens()){context.write(new Text(tokenizer.nextToken()), new IntWritable(1));}}}public static class MyReducer extends Reducer<Text, IntWritable, Text, IntWritable>{@Overrideprotected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {int sum = 0;for (IntWritable v: values) {sum += v.get();}context.write(key, new IntWritable(sum));}}public static void main(String[] args) throws Exception {final String uri = "hdfs://192.168.150.128:9000";Configuration conf = new Configuration();if(System.getProperty("os.name").toLowerCase().contains("win"))conf.set("mapreduce.app-submission.cross-platform","true");Job job = new Job(conf, "word count");String jar = ".\\out\\artifacts\\hadoop_test_jar\\hadoop-test.jar";job.setJar(jar);job.setJarByClass(WordCount.class);job.setMapperClass(MyMapper.class);job.setCombinerClass(MyReducer.class);job.setReducerClass(MyReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);FileInputFormat.addInputPath(job, new Path(uri + "/input"));FileOutputFormat.setOutputPath(job, new Path(uri + "/output"));System.exit(job.waitForCompletion(true) ? 0 : 1);}}

（2）core-site.xml

<configuration><property><name>fs.defaultFS</name><value>hdfs://hadoop01:9000</value>       </property><property><name>hadoop.tmp.dir</name><value>/home/mk/data/tmp</value></property>
</configuration>

（3）mapreduce-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration><property><name>mapreduce.framework.name</name><value>yarn</value>
</property>
<property><name>mapreduce.jobhistory.address</name><value>hadoop01:10020</value>
</property>
<property><name>mapreduce.jobhistory.webapp.address</name><value>hadoop01:19888</value></property></configuration>

（4）yarn-site.xml

<configuration><property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name> <value>org.apache.hadoop.mapred.ShuffleHandler</value> </property> <property><name>yarn.resourcemanager.scheduler.address</name> <value>hadoop01:8030</value> </property> <property> <name>yarn.resourcemanager.resource-tracker.address</name> <value>hadoop01:8025</value> </property><property> <name>yarn.resourcemanager.address</name><value>hadoop01:8040</value></property>
</configuration>

（5）log4j.properties

log4j.rootLogger=INFO, stdout
#log4j.logger.org.springframework=INFO
#log4j.logger.org.apache.activemq=INFO
#log4j.logger.org.apache.activemq.spring=WARN
#log4j.logger.org.apache.activemq.store.journal=INFO
#log4j.logger.org.activeio.journal=INFOlog4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} | %-5.5p | %-16.16t | %-32.32c{1} | %-32.32C %4L | %m%n

（6）pom.xml

<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>com.mk</groupId><artifactId>hadoop-test</artifactId><version>1.0-SNAPSHOT</version><name>hadoop-test</name><url>http://www.mk.com</url><properties><project.build.sourceEncoding>UTF-8</project.build.sourceEncoding><maven.compiler.source>1.7</maven.compiler.source><maven.compiler.target>1.7</maven.compiler.target><project.build.sourceEncoding>UTF-8</project.build.sourceEncoding><hadoop.version>2.6.0</hadoop.version></properties><dependencies><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-client</artifactId><version>${hadoop.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-common</artifactId><version>${hadoop.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-hdfs</artifactId><version>${hadoop.version}</version></dependency><dependency><groupId>junit</groupId><artifactId>junit</artifactId><version>4.11</version><scope>test</scope></dependency></dependencies><build><pluginManagement><plugins><plugin><artifactId>maven-clean-plugin</artifactId><version>3.1.0</version></plugin><plugin><artifactId>maven-resources-plugin</artifactId><version>3.0.2</version></plugin><plugin><artifactId>maven-compiler-plugin</artifactId><version>3.8.0</version></plugin><plugin><artifactId>maven-surefire-plugin</artifactId><version>2.22.1</version></plugin><plugin><artifactId>maven-jar-plugin</artifactId><version>3.0.2</version></plugin><plugin><artifactId>maven-install-plugin</artifactId><version>2.5.2</version></plugin><plugin><artifactId>maven-deploy-plugin</artifactId><version>2.8.2</version></plugin><!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle --><plugin><artifactId>maven-site-plugin</artifactId><version>3.7.1</version></plugin><plugin><artifactId>maven-project-info-reports-plugin</artifactId><version>3.0.0</version></plugin></plugins></pluginManagement></build>
</project>

Hadoop入门（十三）远程提交wordCout程序到hadoop集群相关推荐

【Flnik】解决提交程序到flink集群的异常：Could not complete snapshot 3 for operator Source...
一. 情况描述之前一直在IDEA中运行Flink程序,程序也都很顺利的跑通.但是当把程序打包发布到集群上运行便遇到了一些情况: bin/flink run -m hadoop102:8081 -c ...
Kubernetes入门：Pod、节点、容器和集群都是什么？
文章来源地址: http://mini.eastday.com/a/180103191329488.html?qid=02263 Kubernetes正迅速成为云计算中部署和管理软件的新标准.不过,K ...
使用yaml文件创建deployment来部署一个应用程序到k8s集群
作者: wucong60 原文: https://blog.csdn.net/wucong60/article/details/81586272 系列链接 Kubernetes系列之一:在Ubuntu ...
Hadoop入门进阶步步高（四）-测试Hadoop
2019独角兽企业重金招聘Python工程师标准>>> 四.测试Hadoop 一个简单的求每年温度最大值的程序. 1.准备两个文本测试数据准备两个名为data1.txt及data2 ...
Hadoop（八）Java程序访问HDFS集群中数据块与查看文件系统
阅读目录(Content) 一.HDFS中数据块概述 1.1.HDFS集群中数据块存放位置 1.2.数据块(data block)简介 1.3.对分布式文件系统中的块进行抽象会带来很多好处二.Jav ...
大数据之-Hadoop完全分布式_集群中SSH配置_远程免密码登录_为集群_群起做准备---大数据之hadoop工作笔记0036
之前我们做集群配置的时候,去连接,别的主机,都需要输入这个主机的密码,很麻烦,做分发的时候,还得多次输入不同主机的密码,是相当麻烦现在我们通过配置SSH来实现,主机之间的免密码登录可以看到hado ...
Hadoop入门系列（二）idea配置hadoop开发环境
主要分两部分:一.idea配置hadoop开发环境.二.idea安装hadoop插件. 一. 1.为了便于各个版本的hadoop获得对应jar包以及其依赖jar包.搭建maven项目是个很好的选择.创 ...
JAVA_OA(十四)番外:JAVAWEB防止表单重复提交的方法整合(包括集群部署)
因为自己要用,所以查找了网络上javaweb项目防止表单重复提交的方法,有些部分不太好找,所以整合后贴出来,首先是孤傲苍狼的一部分博客文章,集群部署的解决方案在后面(注意大红字) 原文出处:孤傲苍狼的 ...
spark on k8s准确完整的测试步骤：提交Pi计算程序至k8s集群
部署流程: 下载spark环境包https://www.apache.org/dyn/closer.lua/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tg ...

Hadoop入门（十三）远程提交wordCout程序到hadoop集群

一、项目结构

二、项目源码

Hadoop入门（十三）远程提交wordCout程序到hadoop集群相关推荐

最新文章

热门文章