package core;import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;import java.util.Arrays;
import java.util.List;/*** Created by xupan on 2017/11/28.*/
public class JavaWordCountCore {public static void main(String[] args) {SparkConf conf = new SparkConf().setMaster("local").setAppName("JavaWordCountCore");JavaSparkContext sc = new JavaSparkContext(conf);JavaRDD rdd = sc.textFile("data/JavaWordCountCore.txt");//进行分割JavaRDD flatMapRDD = rdd.flatMap(new FlatMapFunction<String, String>() {public Iterable call(String line) throws Exception {if (StringUtils.isNotEmpty(line)) {return Arrays.asList(line.split(" "));}return null;}});//转换成key/valueJavaPairRDD mapRDD = flatMapRDD.mapToPair(new PairFunction<String, String, Integer>() {public Tuple2<String, Integer> call(String word) throws Exception {return new Tuple2<String, Integer>(word, 1);}});//reduceByKeyJavaPairRDD javaPairRDD = mapRDD.reduceByKey(new Function2<Integer, Integer, Integer>() {public Integer call(final Integer v1, final Integer v2) {return v1 + v2;}});List<Tuple2<String,Integer>> list = javaPairRDD.collect();for (Tuple2<String,Integer> t : list ){System.out.println(t);}}}

(alibaba,1)
(scala,4)
(spark,2)
(hadoop,2)
(eee,1)
(google,1)
(java,1)
(reduce,1)