java实现spark常用算子之ReduceByKey

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Tuple2;

import java.util.Arrays;
import java.util.List;

/**
 * reduceByKey(fun,[numTasks]) 算子:
 * 根据key将value聚合,然后根据fun进行计算
 * 可以设置并行度
 * reduceByKey = groupByKey+reduce
 */
public class ReduceByKeyOperator {
    public static void main(String[] args){
        SparkConf conf = new SparkConf().setMaster("local").setAppName("reduceByKey");
        JavaSparkContext sc = new JavaSparkContext(conf);

        List<Tuple2<String,Integer>> list = Arrays.asList(
                new Tuple2<String,Integer>("w1",1),
                new Tuple2<String,Integer>("w2",2),
                new Tuple2<String,Integer>("w3",3),
                new Tuple2<String,Integer>("w2",22),
                new Tuple2<String,Integer>("w1",11)
        );

        JavaPairRDD<String,Integer> pairRdd = sc.parallelizePairs(list);

        JavaPairRDD<String,Integer> result = pairRdd.reduceByKey(new Function2<Integer, Integer, Integer>() {
            @Override
            public Integer call(Integer integer, Integer integer2) throws Exception {
                return integer+integer2;
            }
        },2);

        result.foreach(new VoidFunction<Tuple2<String, Integer>>() {
            @Override
            public void call(Tuple2<String, Integer> stringIntegerTuple2) throws Exception {
                System.err.println(stringIntegerTuple2._1+":"+stringIntegerTuple2._2);
            }
        });

    }
}

微信扫描下图二维码加入博主知识星球,获取更多大数据、人工智能、算法等免费学习资料哦！