二度人脉推荐(案例)

- 基于:hadoop2.x集群:HDFS + MapReduce

JobFriends
Mao01
Resource01
Map02
Resource02

JobFriends

package com.friend;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class JobFriends {public static void main(String[] args) {Boolean flag = jobOne();if(flag) {jobTwo();}}static Boolean jobOne() {Configuration config =new Configuration();config.set("fs.defaultFS", "hdfs://node01:8020");config.set("yarn.resourcemanager.hostname", "node03:8088");Boolean flag = false;try {Job job = Job.getInstance(config);job.setJarByClass(JobFriends.class);job.setJobName("fof one job");job.setMapperClass(Map01.class);job.setReducerClass(Reduce01.class);job.setMapOutputKeyClass(FoF.class);job.setMapOutputValueClass(IntWritable.class);FileInputFormat.addInputPath(job, new Path("/JF/input/qq.txt"));Path output =new Path("/JF/tuijian/01/");FileSystem fs = FileSystem.get(config);if (fs.exists(output)) {fs.delete(output, true);}FileOutputFormat.setOutputPath(job, output);flag = job.waitForCompletion(true);if (flag) {System.out.println("job 1 success~~");}} catch (Exception e) {e.printStackTrace();};return flag;}static Boolean jobTwo() {  Configuration config =new Configuration();config.set("fs.defaultFS", "hdfs://node01:8020");config.set("yarn.resourcemanager.hostname", "node03:8088");Boolean flag = false;try {Job job = Job.getInstance(config);job.setJarByClass(JobFriends.class);job.setJobName("fof two job");job.setMapperClass(Map02.class);job.setReducerClass(Reduce02.class);job.setMapOutputKeyClass(FriendSort.class);job.setMapOutputValueClass(IntWritable.class);FileInputFormat.addInputPath(job, new Path("/JF/tuijian/01/"));Path output = new Path("/JF/tuijian/02/");FileSystem fs = FileSystem.get(config);if (fs.exists(output)) {fs.delete(output, true);}FileOutputFormat.setOutputPath(job, output);flag = job.waitForCompletion(true);if (flag) {System.out.println("job 2 success~~");}} catch (Exception e) {e.printStackTrace();};return flag;}
}

Map01

package com.friend;import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.util.StringUtils;//输入行号+数据 ,输出map+gongtonghaoy
public class Map01 extends Mapper<LongWritable, Text, FoF, IntWritable>{@Overrideprotected void map(LongWritable key, Text value, Context context)throws java.io.IOException ,InterruptedException {String line = value.toString(); //读取一行 tom   hadoop hive cat helloString[] friends = StringUtils.split(line,'\t');//直接好友关系for(int i = 1; i<friends.length; i++){ //从hadoop hive cat helloString friend = friends[i]; //用户的好友context.write(new FoF(friends[0],friends[i]),  new IntWritable(0));//用户和用户的每个好友  tom hive  tom cat//间接好友关系for(int j = i+1; j<friends.length; j++ ){//遍历 tom hadoop之后的每个好友 hive cat helloString friend2 = friends[j];// hivecontext.write(new FoF(friend,friend2),new IntWritable(1));}}}
}
//tom hadoop 0
//hadoop hive 1
//hadoop cat 1
//hadoop hello 1
//tom hive 0
//hive cat 1
//hive hello 1
//tom cat 0
//cat hello 1

Reduce01

package com.friend;import java.io.IOException;import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.util.StringUtils;
import org.jboss.netty.util.internal.StringUtil;//key ,value : tom hadoop {1,1,1,1,1}
public class Reduce01 extends Reducer<FoF, IntWritable, Text, NullWritable> {@Overrideprotected void reduce(FoF key, Iterable<IntWritable> value,Context context) throws IOException, InterruptedException {int sum = 0;boolean f = true;for(IntWritable i : value){if(i.get() == 0){ //如果是直接好友关系f = false;break;}sum+=i.get();//如果是间接好友关系,计算共同好友数量1+1+1+1=4  }if(f){//如果是间接好友关系 String msg = StringUtils.split(key.toString(),'\t')[0]+" "+ StringUtil.split(key.toString(),'\t')[1] +" " + sum;//将制表符换成空格,加上共同好友数量//tom hadoop 4context.write(new Text(msg), NullWritable.get());}}
}

Map02

package com.friend;import java.io.IOException;import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;public class Map02 extends Mapper<LongWritable, Text, FriendSort, IntWritable>{@Overrideprotected void map(LongWritable key, Text value,Context context)throws IOException, InterruptedException {String lines = value.toString(); //tom hadoop 4String friend1 = StringUtils.split(lines,' ')[0];String friend2 = StringUtils.split(lines,' ')[1];int hot = Integer.parseInt(StringUtils.split(lines, ' ')[2]);System.out.println(friend1+" "+friend2+" "+hot);System.out.println(friend2+" "+friend1+" "+hot);//按用户名tom进行排序,再按用户共同好友数进行排序 tom hadoop 4/ tom hive 2context.write(new FriendSort(friend1,friend2,hot),new IntWritable(hot));System.out.println(friend1+"-"+friend2+"-"+hot);context.write(new FriendSort(friend2,friend1,hot),new IntWritable(hot));}
}

Reduce02

package com.friend;import java.io.IOException;import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;public class Reduce02 extends Reducer<FriendSort, IntWritable, Text, NullWritable>{@Overrideprotected void reduce(FriendSort friend, Iterable<IntWritable> hot,Context context)throws IOException, InterruptedException {int sum = 0;for(IntWritable i : hot){sum=i.get();}System.out.println("==============");String msg = friend.getFriend01()+" "+friend.getFriend02()+" "+sum;System.out.println(msg);context.write(new Text(msg), NullWritable.get());}
}

FoF

package com.friend;import org.apache.hadoop.io.Text;//对key用户名做排序    a-b b-a  统一处理成a-b ->
public class FoF extends Text{public FoF(){super();}public FoF(String friend01,String friend02){set(getof(friend01,friend02));}public String getof(String friend01,String friend02){int c = friend01.compareTo(friend02);if(c > 0){return friend02+"\t"+friend01;}return friend01+"\t"+friend02;}
}

FriendSort

package com.friend;import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;import org.apache.hadoop.io.WritableComparable;//实现WritableComparable比较器接口
public class FriendSort implements WritableComparable<FriendSort>{private String friend01;private String friend02;private int hot;public FriendSort() {super();}public FriendSort(String friend01, String friend02, int hot) {super();this.friend01 = friend01;this.friend02 = friend02;this.hot = hot;}public String getFriend01() {return friend01;}public void setFriend01(String friend01) {this.friend01 = friend01;}public String getFriend02() {return friend02;}public void setFriend02(String friend02) {this.friend02 = friend02;}public int getHot() {return hot;}public void setHot(int hot) {this.hot = hot;}public void readFields(DataInput in) throws IOException {this.friend01=in.readUTF();this.friend02=in.readUTF();this.hot=in.readInt();}public void write(DataOutput out) throws IOException {out.writeUTF(friend01);out.writeUTF(friend02);out.writeInt(hot);}public int compareTo(FriendSort friend) {int c =friend01.compareTo(friend.getFriend01());//下一阶段的用户名与该阶段用户名是否相同,是否是同一用户if(c==0){return -Integer.compare(hot, friend.getHot());//同一用户,比较共同好友数量 tom hadoop 4 tom hive 2,对好友数量进行倒叙排序}return c;}}

二度人脉推荐(案例)相关推荐

Hadoop实例：二度人脉与好友推荐
一.在新浪微博.人人网等社交网站上,为了使用户在网络上认识更多的朋友,社交网站往往提供类似"你可能感兴趣的人"."间接关注推荐"等好友推荐的功能.一直很好奇这个 ...
图数据库Neo4j实现人脉推荐——二度人脉
"吾尝终日而思矣,不如须臾之所学也:吾尝跂而望矣,不如登高之博见也.登高而招,臂非加长也,而见者远:顺风而呼,声非加疾也,而闻者彰.假舆马者,非利足也,而致千里:假舟楫者,非能水也,而绝江河 ...
【算法题】MapReduce编程，寻找二度人脉
这是某资讯APP公司的面试题,考察MapReduce的编程思想. 给定一个人脉关系的文件,从中找到二度人脉.比如给定如下的人脉关系, A B C D E B E F C G G H I J 应输出 A ...
PHP实现二度人脉算法
<?php /** * 数据库中得到的关注列表 * $str字符串中,每个逗号分隔的记录表示前者关注后者 */ $str = "A-B,A-C,A-D,A-E,A-F,A-G,B ...
SDNU1129.多度人脉
Description 艾斯蒂恩优公司最近打算推出一款社交网站,为了让更多用户之间互加好友,项目经理ZZK准备推出多度人脉功能.我们知道,人脉是我们直接的好友,那么多度人脉就是包括好友的好友以及好友的 ...
LinkedIn领英人脉显示1度、2度、3度、领英会员的意思和区别是什么？
对于新注册使用LinkedIn领英的人来说,LinkedIn领英人脉的规则一定是最让人头痛的问题之一,因为我刚开始用LinkedIn领英的时候,就是被这个折磨到差点放弃. 但LinkedIn领英又是全 ...
如何打造领英朋友圈_有哪些领英快速扩充人脉的技巧？
为什么在LinkedIn领英上搜到的客户都是显示领英会员(Linkedin Member)?无法向对方发送添加好友邀请?也无法向访问对方的LinkedIn领英个人主页?这是很多人都遇到的难题,也是使用 ...
大家一起讨论一下朋友网的人脉关系算法是怎么实现的
大家一起讨论一下啊!最短路径? 1.一度人脉:双方直接是好友 2.二度人脉:双方有一个以上共同的好友,这时朋友网可以计算出你们有几个共同的好友并且呈现数字给你.你们的关系是: 你->朋友-> ...
新注册的领英账号如何快速拓展人脉？置顶推荐
刚接触领英不久的新手,甚至老手都会被这个问题困扰.刚注册的Linkedin账号,如何快速拓展人脉?有些人注册后就是一顿疯狂添加.其实这种操作是得不偿失的,这样不仅添加到不好友,而且还会被Linkedi ...

二度人脉推荐(案例)

- 基于:hadoop2.x集群:HDFS + MapReduce

JobFriends

Map01

Reduce01

Map02

Reduce02

FoF

FriendSort

二度人脉推荐(案例)相关推荐

最新文章

热门文章