spark cogroup算子

java

 /**
  *cogroup与join算子不同的是如果rdd中的一个key,对应多个value,则返回<Iterable<key>,Iterable<value>>
  *@author Tele
  */
 public class CogroupDemo {
     private static SparkConf conf = new SparkConf().setMaster("local").setAppName("congroupdemo");
     private static JavaSparkContext jsc = new JavaSparkContext(conf);
     public static void main(String[] args) {
         //每个学生有多门成绩
         List<Tuple2<Integer,String>> studentList = Arrays.asList(
                                                     new Tuple2<Integer,String>(1,"tele"),
                                                     new Tuple2<Integer,String>(1,"xx"),
                                                     new Tuple2<Integer,String>(2,"yeye"),
                                                     new Tuple2<Integer,String>(3,"wyc")
                                                    );

         List<Tuple2<Integer,Integer>> scoreList = Arrays.asList(
                                                   new Tuple2<Integer,Integer>(1,100),
                                                   new Tuple2<Integer,Integer>(1,110),
                                                   new Tuple2<Integer,Integer>(1,120),
                                                   new Tuple2<Integer,Integer>(2,90),
                                                   new Tuple2<Integer,Integer>(2,60),
                                                   new Tuple2<Integer,Integer>(2,50),
                                                   new Tuple2<Integer,Integer>(3,70),
                                                   new Tuple2<Integer,Integer>(3,70)
                                                   );

         JavaPairRDD<Integer, String> studentRDD = jsc.parallelizePairs(studentList);
         JavaPairRDD<Integer, Integer> scoreRDD = jsc.parallelizePairs(scoreList);

         JavaPairRDD<Integer, Tuple2<Iterable<String>, Iterable<Integer>>> result = studentRDD.cogroup(scoreRDD);
         result.foreach(new VoidFunction<Tuple2<Integer,Tuple2<Iterable<String>,Iterable<Integer>>>>() {

             private static final long serialVersionUID = 1L;

             @Override
             public void call(Tuple2<Integer, Tuple2<Iterable<String>, Iterable<Integer>>> t) throws Exception {
                 System.out.println("学号:" + t._1);
                 System.out.println("姓名:" + t._2._1);
                 System.out.println("成绩:" + t._2._2);

             /*    System.out.print("成绩:[");
                 t._2._2.forEach(i->System.out.print(i + ","));
                 System.out.println("]");
                 System.out.println("====================");*/

             }
         });

         jsc.close();
     }
 }

scala

 object CogroupDemo {
     def main(args: Array[String]): Unit = {
         val conf = new SparkConf().setMaster("local").setAppName("cogroupdemo");
         val sc = new SparkContext(conf);

         val studentArr = Array((1,"tele"),(2,"yeye"),(3,"wyc"));
         val scoreArr = Array((1,100),(1,200),(2,80),(2,300),(3,100));

         val studentRDD = sc.parallelize(studentArr,1);
         val scoreRDD = sc.parallelize(scoreArr,1);

         val result = studentRDD.cogroup(scoreRDD);
         result.foreach(t=>{
           println("学号:" + t._1);
           println("姓名:" + t._2._1.mkString(" "));
           println("成绩:" + t._2._2.mkString(","));
           println("============");
         })
     }
 }