【尚学堂·Hadoop学习】MapReduce案例2--好友推荐

案例描述

　　根据好友列表，推荐好友的好友

　　

数据集

tom hello hadoop cat
world hadoop hello hive
cat tom hive
mr hive hello
hive cat hadoop world hello mr
hadoop tom hive world
hello tom world hive mr

代码

　　MyFOF.class

package com.hadoop.mr.fof;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class MyFOF {

    public static void main(String[] args) {

        try {
            //Conf
            Configuration conf = new Configuration(true);
            Job job = Job.getInstance(conf);
            job.setJarByClass(MyFOF.class);

            //Map
            job.setMapperClass(FMapper.class);
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(IntWritable.class);

            //Reduce
            job.setReducerClass(FReducer.class);

            //Input&Output
            Path in = new Path("/user/hadoop/input/friends.txt");
            FileInputFormat.addInputPath(job, in);

            Path out = new Path("/user/hadoop/output/friends/");
            if(out.getFileSystem(conf).exists(out)){
                out.getFileSystem(conf).delete(out,true);
            }
            FileOutputFormat.setOutputPath(job, out);

            //Submit
            job.waitForCompletion(true);

        } catch (IOException e) {
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}

　　FMapper.class

package com.hadoop.mr.fof;

import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.util.StringUtils;

public class FMapper extends Mapper<LongWritable, Text, Text, IntWritable> {

    Text mkey = new Text();
    IntWritable mval = new IntWritable();

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //tom hello hadoop cat

        String [] strs =  StringUtils.split(value.toString(),' ');

        /*
         * 找直接、间接关系
         * value: 0-直接关系;1-间接关系
         * 直接关系：tom:hello tom:hadoop tom:cat
         * 间接关系：hello:hadoop hello:cat hadoop:cat
         */
        for(int i=1;i<strs.length;i++){
            //与好友清单中的好友为直接关系
            mkey.set(getFof(strs[0], strs[i]));
            mval.set(0);
            context.write(mkey, mval);
            //在好友列表内  好友之间为间接关系
            for(int j = i+1;j < strs.length;j++){
                mkey.set(getFof(strs[i],strs[j]));
                mval.set(1);
                context.write(mkey, mval);
            }
        }

    }

    //按字典序进行字符串拼接
    public static String getFof(String s1,String s2){
        if(s1.compareTo(s2) < 0){
            return s1+":"+s2;
        }
        return s2+":"+s1;
    }
}

　　FReducer.class

package com.hadoop.mr.fof;

import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class FReducer extends Reducer<Text,IntWritable, Text, IntWritable> {

    IntWritable rval = new IntWritable();

    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {

        /*
         * 数据如下：
         * hello:hadoop 0
         * hello:hadoop 1
         */

        int flg = 0;    //标志
        int sum = 0;    //共同好友总数
        for(IntWritable v :values){
            if(v.get() == 0){
                flg = 1;
            }
            sum += v.get();
        }

        if(flg == 0){
            rval.set(sum);
            context.write(key, rval);
        }

    }
}

运行结果

　　本次案例只是处理了两个人之间共同好友的数量。