1 数据

1,4
2,5
3,4
2,5
2,4
3,4
2,6

2 需求

统计x轴上每个点线段重叠的次数

3 代码实现

package com._51doit.mr.line;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;/*** Author:   多易教育-行哥* Date:     2020/7/10* Description:*/
public class LineDemo {static class LineMapper extends Mapper<LongWritable, Text, Text, IntWritable> {Text k = new Text() ;IntWritable v = new IntWritable(1) ;/*** map方法每行执行一次* @param key* @param value* @param context* @throws IOException* @throws InterruptedException*/@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {//处理每行  2,5 --> 2,1  3,1  4,1  5,1String[] split = value.toString().split(",");String x1 = split[0];String x2 = split[1];int a1 = Integer.parseInt(x1);int a2 = Integer.parseInt(x2);for (int x = a1; x <= a2; x++) {k.set(x+""); // 更新  不用创建大量的Text// context.write(new Text(x+""),new IntWritable(1));context.write(k,v);}}}static class LineReducer extends Reducer<Text,IntWritable ,Text ,IntWritable> {IntWritable v =new IntWritable() ;@Overrideprotected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {int count = 0;for (IntWritable value : values) {count++ ;}v.set(count);context.write(key,v);}}public static void main(String[] args) throws Exception {Configuration conf = new Configuration();// 参数2  job的名字Job job = Job.getInstance(conf, new LineDemo().getClass().getSimpleName());job.setMapperClass(LineMapper.class);job.setReducerClass(LineReducer.class);// 设置map阶段的输出类型//job.setMapOutputKeyClass(Text.class);// job.setMapOutputValueClass(IntWritable.class);// 最终结果的数据类型job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);job.setNumReduceTasks(2);  //启动3个reduce任务// 待处理数据的路径FileInputFormat.setInputPaths(job, new Path("D:\\data\\line\\input\\"));FileOutputFormat.setOutputPath(job,new Path("D:\\data\\line\\res4\\"));job.waitForCompletion(true) ;}
}

4 知识点

4.1 添加log4j日志配置文件

# Set root logger level to DEBUG and its only appender to A1.
log4j.rootLogger=INFO, A1
# A1 is set to be a ConsoleAppender.
log4j.appender.A1=org.apache.log4j.ConsoleAppender
# A1 uses PatternLayout.
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

4.2 控制台观察日志

D:\tools\jdk1.8.0_65\bin\java.exe "-javaagent:D:\apps\IntelliJ IDEA 2020_01\IntelliJ IDEA 2020.1.3\lib\idea_rt.jar=12052:D:\apps\IntelliJ IDEA 2020_01\IntelliJ IDEA 2020.1.3\bin" -Dfile.encoding=UTF-8 -classpath D:\tools\jdk1.8.0_65\jre\lib\charsets.jar;D:\tools\jdk1.8.0_65\jre\lib\deploy.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\access-bridge-64.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\cldrdata.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\dnsns.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\jaccess.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\jfxrt.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\localedata.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\nashorn.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\sunec.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\sunjce_provider.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\sunmscapi.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\sunpkcs11.jar;D:\tools\jdk1.8.0_65\jre\lib\ext\zipfs.jar;D:\tools\jdk1.8.0_65\jre\lib\javaws.jar;D:\tools\jdk1.8.0_65\jre\lib\jce.jar;D:\tools\jdk1.8.0_65\jre\lib\jfr.jar;D:\tools\jdk1.8.0_65\jre\lib\jfxswt.jar;D:\tools\jdk1.8.0_65\jre\lib\jsse.jar;D:\tools\jdk1.8.0_65\jre\lib\management-agent.jar;D:\tools\jdk1.8.0_65\jre\lib\plugin.jar;D:\tools\jdk1.8.0_65\jre\lib\resources.jar;D:\tools\jdk1.8.0_65\jre\lib\rt.jar;F:\codes\doit16\hdp\target\classes;D:\doit16-maven\repository\org\apache\hadoop\hadoop-common\2.8.5\hadoop-common-2.8.5.jar;D:\doit16-maven\repository\org\apache\hadoop\hadoop-annotations\2.8.5\hadoop-annotations-2.8.5.jar;D:\doit16-maven\repository\com\google\guava\guava\11.0.2\guava-11.0.2.jar;D:\doit16-maven\repository\commons-cli\commons-cli\1.2\commons-cli-1.2.jar;D:\doit16-maven\repository\org\apache\commons\commons-math3\3.1.1\commons-math3-3.1.1.jar;D:\doit16-maven\repository\xmlenc\xmlenc\0.52\xmlenc-0.52.jar;D:\doit16-maven\repository\org\apache\httpcomponents\httpclient\4.5.2\httpclient-4.5.2.jar;D:\doit16-maven\repository\org\apache\httpcomponents\httpcore\4.4.4\httpcore-4.4.4.jar;D:\doit16-maven\repository\commons-codec\commons-codec\1.4\commons-codec-1.4.jar;D:\doit16-maven\repository\commons-io\commons-io\2.4\commons-io-2.4.jar;D:\doit16-maven\repository\commons-net\commons-net\3.1\commons-net-3.1.jar;D:\doit16-maven\repository\commons-collections\commons-collections\3.2.2\commons-collections-3.2.2.jar;D:\doit16-maven\repository\javax\servlet\servlet-api\2.5\servlet-api-2.5.jar;D:\doit16-maven\repository\org\mortbay\jetty\jetty\6.1.26\jetty-6.1.26.jar;D:\doit16-maven\repository\org\mortbay\jetty\jetty-util\6.1.26\jetty-util-6.1.26.jar;D:\doit16-maven\repository\org\mortbay\jetty\jetty-sslengine\6.1.26\jetty-sslengine-6.1.26.jar;D:\doit16-maven\repository\javax\servlet\jsp\jsp-api\2.1\jsp-api-2.1.jar;D:\doit16-maven\repository\com\sun\jersey\jersey-core\1.9\jersey-core-1.9.jar;D:\doit16-maven\repository\com\sun\jersey\jersey-json\1.9\jersey-json-1.9.jar;D:\doit16-maven\repository\org\codehaus\jettison\jettison\1.1\jettison-1.1.jar;D:\doit16-maven\repository\com\sun\xml\bind\jaxb-impl\2.2.3-1\jaxb-impl-2.2.3-1.jar;D:\doit16-maven\repository\org\codehaus\jackson\jackson-jaxrs\1.8.3\jackson-jaxrs-1.8.3.jar;D:\doit16-maven\repository\org\codehaus\jackson\jackson-xc\1.8.3\jackson-xc-1.8.3.jar;D:\doit16-maven\repository\com\sun\jersey\jersey-server\1.9\jersey-server-1.9.jar;D:\doit16-maven\repository\asm\asm\3.1\asm-3.1.jar;D:\doit16-maven\repository\commons-logging\commons-logging\1.1.3\commons-logging-1.1.3.jar;D:\doit16-maven\repository\net\java\dev\jets3t\jets3t\0.9.0\jets3t-0.9.0.jar;D:\doit16-maven\repository\com\jamesmurty\utils\java-xmlbuilder\0.4\java-xmlbuilder-0.4.jar;D:\doit16-maven\repository\commons-lang\commons-lang\2.6\commons-lang-2.6.jar;D:\doit16-maven\repository\commons-configuration\commons-configuration\1.6\commons-configuration-1.6.jar;D:\doit16-maven\repository\commons-digester\commons-digester\1.8\commons-digester-1.8.jar;D:\doit16-maven\repository\commons-beanutils\commons-beanutils\1.7.0\commons-beanutils-1.7.0.jar;D:\doit16-maven\repository\commons-beanutils\commons-beanutils-core\1.8.0\commons-beanutils-core-1.8.0.jar;D:\doit16-maven\repository\org\slf4j\slf4j-api\1.7.10\slf4j-api-1.7.10.jar;D:\doit16-maven\repository\org\slf4j\slf4j-log4j12\1.7.10\slf4j-log4j12-1.7.10.jar;D:\doit16-maven\repository\org\codehaus\jackson\jackson-core-asl\1.9.13\jackson-core-asl-1.9.13.jar;D:\doit16-maven\repository\org\codehaus\jackson\jackson-mapper-asl\1.9.13\jackson-mapper-asl-1.9.13.jar;D:\doit16-maven\repository\org\apache\avro\avro\1.7.4\avro-1.7.4.jar;D:\doit16-maven\repository\com\thoughtworks\paranamer\paranamer\2.3\paranamer-2.3.jar;D:\doit16-maven\repository\org\xerial\snappy\snappy-java\1.0.4.1\snappy-java-1.0.4.1.jar;D:\doit16-maven\repository\com\google\protobuf\protobuf-java\2.5.0\protobuf-java-2.5.0.jar;D:\doit16-maven\repository\com\google\code\gson\gson\2.2.4\gson-2.2.4.jar;D:\doit16-maven\repository\org\apache\hadoop\hadoop-auth\2.8.5\hadoop-auth-2.8.5.jar;D:\doit16-maven\repository\com\nimbusds\nimbus-jose-jwt\4.41.1\nimbus-jose-jwt-4.41.1.jar;D:\doit16-maven\repository\com\github\stephenc\jcip\jcip-annotations\1.0-1\jcip-annotations-1.0-1.jar;D:\doit16-maven\repository\net\minidev\json-smart\2.3\json-smart-2.3.jar;D:\doit16-maven\repository\net\minidev\accessors-smart\1.2\accessors-smart-1.2.jar;D:\doit16-maven\repository\org\ow2\asm\asm\5.0.4\asm-5.0.4.jar;D:\doit16-maven\repository\org\apache\directory\server\apacheds-kerberos-codec\2.0.0-M15\apacheds-kerberos-codec-2.0.0-M15.jar;D:\doit16-maven\repository\org\apache\directory\server\apacheds-i18n\2.0.0-M15\apacheds-i18n-2.0.0-M15.jar;D:\doit16-maven\repository\org\apache\directory\api\api-asn1-api\1.0.0-M20\api-asn1-api-1.0.0-M20.jar;D:\doit16-maven\repository\org\apache\directory\api\api-util\1.0.0-M20\api-util-1.0.0-M20.jar;D:\doit16-maven\repository\org\apache\curator\curator-framework\2.7.1\curator-framework-2.7.1.jar;D:\doit16-maven\repository\com\jcraft\jsch\0.1.54\jsch-0.1.54.jar;D:\doit16-maven\repository\org\apache\curator\curator-client\2.7.1\curator-client-2.7.1.jar;D:\doit16-maven\repository\org\apache\curator\curator-recipes\2.7.1\curator-recipes-2.7.1.jar;D:\doit16-maven\repository\com\google\code\findbugs\jsr305\3.0.0\jsr305-3.0.0.jar;D:\doit16-maven\repository\org\apache\htrace\htrace-core4\4.0.1-incubating\htrace-core4-4.0.1-incubating.jar;D:\doit16-maven\repository\org\apache\zookeeper\zookeeper\3.4.6\zookeeper-3.4.6.jar;D:\doit16-maven\repository\org\apache\commons\commons-compress\1.4.1\commons-compress-1.4.1.jar;D:\doit16-maven\repository\org\tukaani\xz\1.0\xz-1.0.jar;D:\doit16-maven\repository\org\apache\hadoop\hadoop-hdfs\2.8.5\hadoop-hdfs-2.8.5.jar;D:\doit16-maven\repository\org\apache\hadoop\hadoop-hdfs-client\2.8.5\hadoop-hdfs-client-2.8.5.jar;D:\doit16-maven\repository\com\squareup\okhttp\okhttp\2.4.0\okhttp-2.4.0.jar;D:\doit16-maven\repository\com\squareup\okio\okio\1.4.0\okio-1.4.0.jar;D:\doit16-maven\repository\commons-daemon\commons-daemon\1.0.13\commons-daemon-1.0.13.jar;D:\doit16-maven\repository\io\netty\netty\3.6.2.Final\netty-3.6.2.Final.jar;D:\doit16-maven\repository\io\netty\netty-all\4.0.23.Final\netty-all-4.0.23.Final.jar;D:\doit16-maven\repository\xerces\xercesImpl\2.9.1\xercesImpl-2.9.1.jar;D:\doit16-maven\repository\xml-apis\xml-apis\1.3.04\xml-apis-1.3.04.jar;D:\doit16-maven\repository\org\fusesource\leveldbjni\leveldbjni-all\1.8\leveldbjni-all-1.8.jar;D:\doit16-maven\repository\org\apache\hadoop\hadoop-mapreduce-client-core\2.8.5\hadoop-mapreduce-client-core-2.8.5.jar;D:\doit16-maven\repository\org\apache\hadoop\hadoop-yarn-common\2.8.5\hadoop-yarn-common-2.8.5.jar;D:\doit16-maven\repository\org\apache\hadoop\hadoop-yarn-api\2.8.5\hadoop-yarn-api-2.8.5.jar;D:\doit16-maven\repository\javax\xml\bind\jaxb-api\2.2.2\jaxb-api-2.2.2.jar;D:\doit16-maven\repository\javax\xml\stream\stax-api\1.0-2\stax-api-1.0-2.jar;D:\doit16-maven\repository\javax\activation\activation\1.1\activation-1.1.jar;D:\doit16-maven\repository\com\sun\jersey\jersey-client\1.9\jersey-client-1.9.jar;D:\doit16-maven\repository\com\google\inject\guice\3.0\guice-3.0.jar;D:\doit16-maven\repository\javax\inject\javax.inject\1\javax.inject-1.jar;D:\doit16-maven\repository\aopalliance\aopalliance\1.0\aopalliance-1.0.jar;D:\doit16-maven\repository\com\sun\jersey\contribs\jersey-guice\1.9\jersey-guice-1.9.jar;D:\doit16-maven\repository\com\google\inject\extensions\guice-servlet\3.0\guice-servlet-3.0.jar;D:\doit16-maven\repository\org\apache\hadoop\hadoop-mapreduce-client-common\2.8.5\hadoop-mapreduce-client-common-2.8.5.jar;D:\doit16-maven\repository\org\apache\hadoop\hadoop-yarn-client\2.8.5\hadoop-yarn-client-2.8.5.jar;D:\doit16-maven\repository\org\apache\hadoop\hadoop-yarn-server-common\2.8.5\hadoop-yarn-server-common-2.8.5.jar;D:\doit16-maven\repository\log4j\log4j\1.2.17\log4j-1.2.17.jar com._51doit.mr.line.LineDemo
0    [main] INFO  org.apache.hadoop.conf.Configuration.deprecation  - session.id is deprecated. Instead, use dfs.metrics.session-id
2    [main] INFO  org.apache.hadoop.metrics.jvm.JvmMetrics  - Initializing JVM Metrics with processName=JobTracker, sessionId=
667  [main] WARN  org.apache.hadoop.mapreduce.JobResourceUploader  - Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
715  [main] WARN  org.apache.hadoop.mapreduce.JobResourceUploader  - No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
953  [main] INFO  org.apache.hadoop.mapreduce.lib.input.FileInputFormat  - Total input files to process : 3
1000 [main] INFO  org.apache.hadoop.mapreduce.JobSubmitter  - number of splits:3
1130 [main] INFO  org.apache.hadoop.mapreduce.JobSubmitter  - Submitting tokens for job: job_local190908434_0001
1542 [main] INFO  org.apache.hadoop.mapreduce.Job  - The url to track the job: http://localhost:8080/
1543 [main] INFO  org.apache.hadoop.mapreduce.Job  - Running job: job_local190908434_0001
1545 [Thread-6] INFO  org.apache.hadoop.mapred.LocalJobRunner  - OutputCommitter set in config null
1550 [Thread-6] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - File Output Committer Algorithm version is 1
1550 [Thread-6] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
1550 [Thread-6] INFO  org.apache.hadoop.mapred.LocalJobRunner  - OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
1595 [Thread-6] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Waiting for map tasks
1596 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Starting task: attempt_local190908434_0001_m_000000_0
1629 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - File Output Committer Algorithm version is 1
1630 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
1640 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.yarn.util.ProcfsBasedProcessTree  - ProcfsBasedProcessTree currently is supported only on Linux.
1683 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  -  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@6c404c50
1691 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Processing split: file:/D:/data/line/input/line - 副本 (2).txt:0+33
1712 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - (EQUATOR) 0 kvi 26214396(104857584)
1712 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - mapreduce.task.io.sort.mb: 100
1712 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - soft limit at 83886080
1713 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - bufstart = 0; bufvoid = 104857600
1713 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - kvstart = 26214396; length = 6553600
1717 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
1726 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - 
1726 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Starting flush of map output
1726 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Spilling map output
1726 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - bufstart = 0; bufend = 144; bufvoid = 104857600
1726 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - kvstart = 26214396(104857584); kvend = 26214304(104857216); length = 93/6553600
1843 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Finished spill 0
1852 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  - Task:attempt_local190908434_0001_m_000000_0 is done. And is in the process of committing
1872 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - map
1872 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  - Task 'attempt_local190908434_0001_m_000000_0' done.
1876 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  - Final Counters for attempt_local190908434_0001_m_000000_0: Counters: 17
    File System Counters
        FILE: Number of bytes read=430
        FILE: Number of bytes written=373873
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
    Map-Reduce Framework
        Map input records=7
        Map output records=24
        Map output bytes=144
        Map output materialized bytes=204
        Input split bytes=111
        Combine input records=0
        Spilled Records=24
        Failed Shuffles=0
        Merged Map outputs=0
        GC time elapsed (ms)=0
        Total committed heap usage (bytes)=514850816
    File Input Format Counters 
        Bytes Read=33
1877 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Finishing task: attempt_local190908434_0001_m_000000_0
1877 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Starting task: attempt_local190908434_0001_m_000001_0
1878 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - File Output Committer Algorithm version is 1
1878 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
1878 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.yarn.util.ProcfsBasedProcessTree  - ProcfsBasedProcessTree currently is supported only on Linux.
1941 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  -  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@7f711871
1943 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Processing split: file:/D:/data/line/input/line - 副本.txt:0+33
1987 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - (EQUATOR) 0 kvi 26214396(104857584)
1987 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - mapreduce.task.io.sort.mb: 100
1987 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - soft limit at 83886080
1987 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - bufstart = 0; bufvoid = 104857600
1987 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - kvstart = 26214396; length = 6553600
1988 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
1990 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - 
1990 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Starting flush of map output
1990 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Spilling map output
1990 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - bufstart = 0; bufend = 144; bufvoid = 104857600
1990 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - kvstart = 26214396(104857584); kvend = 26214304(104857216); length = 93/6553600
1997 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Finished spill 0
2006 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  - Task:attempt_local190908434_0001_m_000001_0 is done. And is in the process of committing
2010 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - map
2010 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  - Task 'attempt_local190908434_0001_m_000001_0' done.
2012 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  - Final Counters for attempt_local190908434_0001_m_000001_0: Counters: 17
    File System Counters
        FILE: Number of bytes read=798
        FILE: Number of bytes written=374133
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
    Map-Reduce Framework
        Map input records=7
        Map output records=24
        Map output bytes=144
        Map output materialized bytes=204
        Input split bytes=107
        Combine input records=0
        Spilled Records=24
        Failed Shuffles=0
        Merged Map outputs=0
        GC time elapsed (ms)=0
        Total committed heap usage (bytes)=514850816
    File Input Format Counters 
        Bytes Read=33
2012 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Finishing task: attempt_local190908434_0001_m_000001_0
2013 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Starting task: attempt_local190908434_0001_m_000002_0
2014 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - File Output Committer Algorithm version is 1
2014 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
2015 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.yarn.util.ProcfsBasedProcessTree  - ProcfsBasedProcessTree currently is supported only on Linux.
2055 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  -  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@2114082e
2057 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Processing split: file:/D:/data/line/input/line.txt:0+33
2103 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - (EQUATOR) 0 kvi 26214396(104857584)
2104 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - mapreduce.task.io.sort.mb: 100
2104 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - soft limit at 83886080
2104 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - bufstart = 0; bufvoid = 104857600
2104 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - kvstart = 26214396; length = 6553600
2111 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
2113 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - 
2114 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Starting flush of map output
2114 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Spilling map output
2114 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - bufstart = 0; bufend = 144; bufvoid = 104857600
2114 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - kvstart = 26214396(104857584); kvend = 26214304(104857216); length = 93/6553600
2220 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.MapTask  - Finished spill 0
2224 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  - Task:attempt_local190908434_0001_m_000002_0 is done. And is in the process of committing
2226 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - map
2226 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  - Task 'attempt_local190908434_0001_m_000002_0' done.
2226 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.Task  - Final Counters for attempt_local190908434_0001_m_000002_0: Counters: 17
    File System Counters
        FILE: Number of bytes read=1166
        FILE: Number of bytes written=374393
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
    Map-Reduce Framework
        Map input records=7
        Map output records=24
        Map output bytes=144
        Map output materialized bytes=204
        Input split bytes=98
        Combine input records=0
        Spilled Records=24
        Failed Shuffles=0
        Merged Map outputs=0
        GC time elapsed (ms)=6
        Total committed heap usage (bytes)=514850816
    File Input Format Counters 
        Bytes Read=33
2226 [LocalJobRunner Map Task Executor #0] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Finishing task: attempt_local190908434_0001_m_000002_0
2226 [Thread-6] INFO  org.apache.hadoop.mapred.LocalJobRunner  - map task executor complete.
2228 [Thread-6] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Waiting for reduce tasks
2229 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Starting task: attempt_local190908434_0001_r_000000_0
2241 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - File Output Committer Algorithm version is 1
2241 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
2241 [pool-4-thread-1] INFO  org.apache.hadoop.yarn.util.ProcfsBasedProcessTree  - ProcfsBasedProcessTree currently is supported only on Linux.
2280 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Task  -  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@41c38664
2282 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.ReduceTask  - Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@25868440
2298 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - MergerManager: memoryLimit=5328129024, maxSingleShuffleLimit=1332032256, mergeThreshold=3516565248, ioSortFactor=10, memToMemMergeOutputsThreshold=10
2300 [EventFetcher for fetching Map Completion Events] INFO  org.apache.hadoop.mapreduce.task.reduce.EventFetcher  - attempt_local190908434_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events
2343 [localfetcher#1] INFO  org.apache.hadoop.mapreduce.task.reduce.LocalFetcher  - localfetcher#1 about to shuffle output of map attempt_local190908434_0001_m_000001_0 decomp: 90 len: 94 to MEMORY
2350 [localfetcher#1] INFO  org.apache.hadoop.mapreduce.task.reduce.InMemoryMapOutput  - Read 90 bytes from map-output for attempt_local190908434_0001_m_000001_0
2351 [localfetcher#1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - closeInMemoryFile -> map-output of size: 90, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->90
2356 [localfetcher#1] INFO  org.apache.hadoop.mapreduce.task.reduce.LocalFetcher  - localfetcher#1 about to shuffle output of map attempt_local190908434_0001_m_000000_0 decomp: 90 len: 94 to MEMORY
2358 [localfetcher#1] INFO  org.apache.hadoop.mapreduce.task.reduce.InMemoryMapOutput  - Read 90 bytes from map-output for attempt_local190908434_0001_m_000000_0
2358 [localfetcher#1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - closeInMemoryFile -> map-output of size: 90, inMemoryMapOutputs.size() -> 2, commitMemory -> 90, usedMemory ->180
2363 [localfetcher#1] INFO  org.apache.hadoop.mapreduce.task.reduce.LocalFetcher  - localfetcher#1 about to shuffle output of map attempt_local190908434_0001_m_000002_0 decomp: 90 len: 94 to MEMORY
2364 [localfetcher#1] INFO  org.apache.hadoop.mapreduce.task.reduce.InMemoryMapOutput  - Read 90 bytes from map-output for attempt_local190908434_0001_m_000002_0
2364 [localfetcher#1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - closeInMemoryFile -> map-output of size: 90, inMemoryMapOutputs.size() -> 3, commitMemory -> 180, usedMemory ->270
2365 [EventFetcher for fetching Map Completion Events] INFO  org.apache.hadoop.mapreduce.task.reduce.EventFetcher  - EventFetcher is interrupted.. Returning
2366 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - 3 / 3 copied.
2366 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - finalMerge called with 3 in-memory map-outputs and 0 on-disk map-outputs
2376 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Merger  - Merging 3 sorted segments
2377 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Merger  - Down to the last merge-pass, with 3 segments left of total size: 258 bytes
2380 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - Merged 3 segments, 270 bytes to disk to satisfy reduce memory limit
2381 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - Merging 1 files, 270 bytes from disk
2382 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - Merging 0 segments, 0 bytes from memory into reduce
2382 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Merger  - Merging 1 sorted segments
2383 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Merger  - Down to the last merge-pass, with 1 segments left of total size: 262 bytes
2384 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - 3 / 3 copied.
2391 [pool-4-thread-1] INFO  org.apache.hadoop.conf.Configuration.deprecation  - mapred.skip.on is deprecated. Instead, use mapreduce.job.skiprecords
2396 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Task  - Task:attempt_local190908434_0001_r_000000_0 is done. And is in the process of committing
2398 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - 3 / 3 copied.
2398 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Task  - Task attempt_local190908434_0001_r_000000_0 is allowed to commit now
2400 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - Saved output of task 'attempt_local190908434_0001_r_000000_0' to file:/D:/data/line/res4/_temporary/0/task_local190908434_0001_r_000000
2401 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - reduce > reduce
2401 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Task  - Task 'attempt_local190908434_0001_r_000000_0' done.
2402 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Task  - Final Counters for attempt_local190908434_0001_r_000000_0: Counters: 24
    File System Counters
        FILE: Number of bytes read=2216
        FILE: Number of bytes written=374688
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
    Map-Reduce Framework
        Combine input records=0
        Combine output records=0
        Reduce input groups=3
        Reduce shuffle bytes=282
        Reduce input records=33
        Reduce output records=3
        Spilled Records=33
        Shuffled Maps =3
        Failed Shuffles=0
        Merged Map outputs=3
        GC time elapsed (ms)=0
        Total committed heap usage (bytes)=514850816
    Shuffle Errors
        BAD_ID=0
        CONNECTION=0
        IO_ERROR=0
        WRONG_LENGTH=0
        WRONG_MAP=0
        WRONG_REDUCE=0
    File Output Format Counters 
        Bytes Written=25
2403 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Finishing task: attempt_local190908434_0001_r_000000_0
2403 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Starting task: attempt_local190908434_0001_r_000001_0
2404 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - File Output Committer Algorithm version is 1
2404 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false
2406 [pool-4-thread-1] INFO  org.apache.hadoop.yarn.util.ProcfsBasedProcessTree  - ProcfsBasedProcessTree currently is supported only on Linux.
2447 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Task  -  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@173b77b0
2447 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.ReduceTask  - Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@1126b874
2447 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - MergerManager: memoryLimit=5328129024, maxSingleShuffleLimit=1332032256, mergeThreshold=3516565248, ioSortFactor=10, memToMemMergeOutputsThreshold=10
2448 [EventFetcher for fetching Map Completion Events] INFO  org.apache.hadoop.mapreduce.task.reduce.EventFetcher  - attempt_local190908434_0001_r_000001_0 Thread started: EventFetcher for fetching Map Completion Events
2453 [localfetcher#2] INFO  org.apache.hadoop.mapreduce.task.reduce.LocalFetcher  - localfetcher#2 about to shuffle output of map attempt_local190908434_0001_m_000001_0 decomp: 106 len: 110 to MEMORY
2454 [localfetcher#2] INFO  org.apache.hadoop.mapreduce.task.reduce.InMemoryMapOutput  - Read 106 bytes from map-output for attempt_local190908434_0001_m_000001_0
2454 [localfetcher#2] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - closeInMemoryFile -> map-output of size: 106, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->106
2458 [localfetcher#2] INFO  org.apache.hadoop.mapreduce.task.reduce.LocalFetcher  - localfetcher#2 about to shuffle output of map attempt_local190908434_0001_m_000000_0 decomp: 106 len: 110 to MEMORY
2458 [localfetcher#2] INFO  org.apache.hadoop.mapreduce.task.reduce.InMemoryMapOutput  - Read 106 bytes from map-output for attempt_local190908434_0001_m_000000_0
2458 [localfetcher#2] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - closeInMemoryFile -> map-output of size: 106, inMemoryMapOutputs.size() -> 2, commitMemory -> 106, usedMemory ->212
2462 [localfetcher#2] INFO  org.apache.hadoop.mapreduce.task.reduce.LocalFetcher  - localfetcher#2 about to shuffle output of map attempt_local190908434_0001_m_000002_0 decomp: 106 len: 110 to MEMORY
2463 [localfetcher#2] INFO  org.apache.hadoop.mapreduce.task.reduce.InMemoryMapOutput  - Read 106 bytes from map-output for attempt_local190908434_0001_m_000002_0
2463 [localfetcher#2] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - closeInMemoryFile -> map-output of size: 106, inMemoryMapOutputs.size() -> 3, commitMemory -> 212, usedMemory ->318
2463 [EventFetcher for fetching Map Completion Events] INFO  org.apache.hadoop.mapreduce.task.reduce.EventFetcher  - EventFetcher is interrupted.. Returning
2464 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - 3 / 3 copied.
2464 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - finalMerge called with 3 in-memory map-outputs and 0 on-disk map-outputs
2470 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Merger  - Merging 3 sorted segments
2470 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Merger  - Down to the last merge-pass, with 3 segments left of total size: 306 bytes
2473 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - Merged 3 segments, 318 bytes to disk to satisfy reduce memory limit
2474 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - Merging 1 files, 318 bytes from disk
2474 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl  - Merging 0 segments, 0 bytes from memory into reduce
2474 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Merger  - Merging 1 sorted segments
2475 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Merger  - Down to the last merge-pass, with 1 segments left of total size: 310 bytes
2475 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - 3 / 3 copied.
2481 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Task  - Task:attempt_local190908434_0001_r_000001_0 is done. And is in the process of committing
2482 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - 3 / 3 copied.
2482 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Task  - Task attempt_local190908434_0001_r_000001_0 is allowed to commit now
2489 [pool-4-thread-1] INFO  org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter  - Saved output of task 'attempt_local190908434_0001_r_000001_0' to file:/D:/data/line/res4/_temporary/0/task_local190908434_0001_r_000001
2490 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - reduce > reduce
2490 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Task  - Task 'attempt_local190908434_0001_r_000001_0' done.
2491 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.Task  - Final Counters for attempt_local190908434_0001_r_000001_0: Counters: 24
    File System Counters
        FILE: Number of bytes read=3032
        FILE: Number of bytes written=375032
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
    Map-Reduce Framework
        Combine input records=0
        Combine output records=0
        Reduce input groups=3
        Reduce shuffle bytes=330
        Reduce input records=39
        Reduce output records=3
        Spilled Records=39
        Shuffled Maps =3
        Failed Shuffles=0
        Merged Map outputs=3
        GC time elapsed (ms)=0
        Total committed heap usage (bytes)=514850816
    Shuffle Errors
        BAD_ID=0
        CONNECTION=0
        IO_ERROR=0
        WRONG_LENGTH=0
        WRONG_MAP=0
        WRONG_REDUCE=0
    File Output Format Counters 
        Bytes Written=26
2491 [pool-4-thread-1] INFO  org.apache.hadoop.mapred.LocalJobRunner  - Finishing task: attempt_local190908434_0001_r_000001_0
2491 [Thread-6] INFO  org.apache.hadoop.mapred.LocalJobRunner  - reduce task executor complete.
2545 [main] INFO  org.apache.hadoop.mapreduce.Job  - Job job_local190908434_0001 running in uber mode : false
2548 [main] INFO  org.apache.hadoop.mapreduce.Job  -  map 100% reduce 100%
2550 [main] INFO  org.apache.hadoop.mapreduce.Job  - Job job_local190908434_0001 completed successfully
2560 [main] INFO  org.apache.hadoop.mapreduce.Job  - Counters: 30
    File System Counters
        FILE: Number of bytes read=7642
        FILE: Number of bytes written=1872119
        FILE: Number of read operations=0
        FILE: Number of large read operations=0
        FILE: Number of write operations=0
    Map-Reduce Framework
        Map input records=21
        Map output records=72
        Map output bytes=432
        Map output materialized bytes=612
        Input split bytes=316
        Combine input records=0
        Combine output records=0
        Reduce input groups=6
        Reduce shuffle bytes=612
        Reduce input records=72
        Reduce output records=6
        Spilled Records=144
        Shuffled Maps =6
        Failed Shuffles=0
        Merged Map outputs=6
        GC time elapsed (ms)=6
        Total committed heap usage (bytes)=2574254080
    Shuffle Errors
        BAD_ID=0
        CONNECTION=0
        IO_ERROR=0
        WRONG_LENGTH=0
        WRONG_MAP=0
        WRONG_REDUCE=0
    File Input Format Counters 
        Bytes Read=99
    File Output Format Counters 
        Bytes Written=51

Process finished with exit code 0

4.3 maptask和reducetask个数

  1. maptask的个数和处理文件的个数以及大小有关  计算逻辑在MR原理加强篇
  2. reducetask的个数可以使用job.setNumReduceTasks(n)设置

hadoop详细笔记(十一) mapreduce数据分析案例之线段重叠案例相关推荐

  1. Hadoop学习笔记—11.MapReduce中的排序和分组

    Hadoop学习笔记-11.MapReduce中的排序和分组 一.写在之前的 1.1 回顾Map阶段四大步骤 首先,我们回顾一下在MapReduce中,排序和分组在哪里被执行: 从上图中可以清楚地看出 ...

  2. scala详细笔记(七)scala集合练习题 [函数练习题][scala案例][scala练习]

    0 经典WorkCount案例 统计文本文件中单词出现的次数 import scala.io.Source import scala.reflect.io.File /*** @Author 多易教育 ...

  3. Hadoop学习笔记：MapReduce框架详解

    原文:http://blog.jobbole.com/84089/ 原文出处: 夏天的森林 开始聊mapreduce,mapreduce是hadoop的计算框架,我学hadoop是从hive开始入手, ...

  4. hadoop 学习笔记：mapreduce框架详解

    开始聊mapreduce,mapreduce是hadoop的计算框架,我学hadoop是从hive开始入手,再到hdfs,当我学习hdfs时候,就感觉到hdfs和mapreduce关系的紧密.这个可能 ...

  5. Hadoop学习笔记—20.网站日志分析项目案例（一）项目介绍

    Hadoop学习笔记-20.网站日志分析项目案例(一)项目介绍 网站日志分析项目案例(一)项目介绍:当前页面 网站日志分析项目案例(二)数据清洗:http://www.cnblogs.com/edis ...

  6. Hadoop学习笔记—4.初识MapReduce

    一.神马是高大上的MapReduce MapReduce是Google的一项重要技术,它首先是一个 编程模型 ,用以进行大数据量的计算.对于大 数据量的计算,通常采用的处理手法就是并行计算.但对许多开 ...

  7. Hadoop阅读笔记（四）——一幅图看透MapReduce机制

    时至今日,已然看到第十章,似乎越是焦躁什么时候能翻完这本圣经的时候也让自己变得更加浮躁,想想后面还有一半的行程没走,我觉得这样"有口无心"的学习方式是不奏效的,或者是收效甚微的.如 ...

  8. Hadoop阅读笔记（一）——强大的MapReduce

    前言:来园子已经有8个月了,当初入园凭着满腔热血和一脑门子冲动,给自己起了个响亮的旗号"大数据 小世界",顿时有了种世界都是我的,世界都在我手中的赶脚.可是......时光飞逝,岁 ...

  9. oracle registers,【案例】Oracle RAC强制删除node节点过程的详细笔记

    天萃荷净 Oracle研究中心案例分析:分享一篇关于Oracle RAC节点间操作的笔记,详细介绍如何强制删除Oracle RAC中的节点. 本站文章除注明转载外,均为本站原创: 转载自love wi ...

最新文章

  1. hdfs.DFSClient: Exception in createBlockOutputStream
  2. Code片段 ： .properties属性文件操作工具类 JSON工具类
  3. 第十章：Java_IO流
  4. 学习教材《构建之法》遇到的问题及思路
  5. mybatis删除成功返回0_你还在用分页？试试 MyBatis 流式查询，真心强大！
  6. 2014年考研英语一完型填空知识点
  7. 满足多个条件的JAVA语句_关于Java：关于具有多个条件的If语句的快速问题
  8. 跑步记录日期怎么改_快捷增加历史记录-鲨鱼记账App功能优化
  9. 从skyeye学习arm（开篇）
  10. 用汇编的眼光看C++（之类静态变量、静态函数）
  11. linux系统管理Linux系统实验,实验三 linux系统管理.doc
  12. C#项目xxx针对的是.NETFramework,Version=vxxx之解决方案
  13. 树莓派学习（二）：摄像头模块的安装和使用
  14. ANSYS 有限元分析 命令流 实例
  15. 矩阵乘法np.dot()及np.multipy()区别
  16. 小学计算机期末评价方案,小学班级管理评价方案
  17. python def是什么意思-python里面def是什么意思
  18. 最流行前端开发框架对比评测
  19. python计算圆周率_Python圆周率计算
  20. DEBIAN_FRONTEND noninteractive参数

热门文章

  1. （附源码）计算机毕业设计SSM流浪动物救助系统
  2. jQuery获取下拉选单的值
  3. MGMT接口采集服务器IMM日志
  4. 在印度与软件相关的发明可不可以申请专利？
  5. Android音乐播放器的比较。
  6. NSString 和 UInt8 相互转换
  7. 单商户商城系统功能拆解47—应用中心—自定义海报
  8. 深度学习中常见的打标签工具和数据集资源
  9. 机器学习:房价预测项目实战
  10. java8 LocalDate获取自然周的方法