spark accumulator累加器

java

 /**
  * accumulator可以让多个task共同操作一份变量,主要进行多个节点对一个变量进行共享性的操作,accumulator只提供了累加的功能
  * 只有driver可以获取accumulator的值
  * @author Tele
  */
 public class AccumulatorDemo {
     private static SparkConf conf = new SparkConf().setMaster("local").setAppName("AccumulatorDemo");
     private static JavaSparkContext jsc = new JavaSparkContext(conf);

     public static void main(String[] args) {
         List<Integer> list = Arrays.asList(1, 2, 3, 4, 5, 6);

         JavaRDD<Integer> rdd = jsc.parallelize(list);

         /*
          * Accumulator<Integer> accumulator = jsc.accumulator(10);
          *
          * rdd.foreach(new VoidFunction<Integer>() {
          *
          * private static final long serialVersionUID = 1L;
          *
          * @Override public void call(Integer t) throws Exception { accumulator.add(t);
          * } }); System.out.println(accumulator.value());
          */

         LongAccumulator la = new LongAccumulator();
         la.setValue(100L);

         jsc.sc().register(la, "数值累加器");

         rdd.foreach(new VoidFunction<Integer>() {

             private static final long serialVersionUID = 1L;

             @Override
             public void call(Integer t) throws Exception {
                 // 不能在算子内部获得accumulator.value()
                 la.add(t);
             }
         });

         System.out.println(la.value());
         jsc.close();
     }
 }

scala

 object AccumulatorDemo {
   def main(args: Array[String]): Unit = {
     val conf = new SparkConf().setMaster("local").setAppName("accumulator");
     val sc = new SparkContext(conf);

     val arr = Array(1, 2, 3, 4, 5);
     val rdd = sc.parallelize(arr, 1);

     val accumulator = new LongAccumulator;
     accumulator.add(100);

     sc.register(accumulator);

     rdd.foreach(accumulator.add(_));

     println(accumulator.value);

   }
 }