Spark累加器

spark累计器

因为task的执行是在多个Executor中执行，所以会出现计算总量的时候，每个Executor只会计算部分数据，不能全局计算。

累计器是可以实现在全局中进行累加计数。

注意：

累加器只能在driver端定义，driver端读取，不能在Executor端读取。

广播变量只能在driver端定义，在Executor端读取，Executor不能修改。

下面是实践的代码：

package SparkStreaming;

import org.apache.commons.collections.iterators.ArrayListIterator;
import org.apache.commons.io.LineIterator;
import org.apache.spark.Accumulator;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;

import java.util.Iterator;
import java.util.List;

public class totalization_device {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf()
                .setMaster("local[2]")
                .setAppName("totalization_device");
        JavaSparkContext sc = new JavaSparkContext(conf);
        /*
        * 定义一个累加器
        * */
        Accumulator<Integer> accumulator = sc.accumulator();
        JavaRDD<String> fileRDD = sc.textFile("E:/2018_cnic/learn/wordcount.txt");
        JavaRDD<String> fileRDD1 = fileRDD.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public Iterator<String> call(String s) throws Exception {
                accumulator.add();
                return new ArrayListIterator(s.split(" "));
            }
        });
        JavaPairRDD<String, Integer> pairRDD = fileRDD1.mapToPair(new PairFunction<String, String, Integer>() {

            @Override
            public Tuple2<String, Integer> call(String s) throws Exception {
                return new Tuple2<String, Integer>(s,);
            }
        });
        JavaPairRDD<String, Integer> reducebykeyRDD = pairRDD.reduceByKey(new Function2<Integer, Integer, Integer>() {
            @Override
            public Integer call(Integer integer, Integer integer2) throws Exception {
                return integer + integer2;
            }
        });
        List<Tuple2<String, Integer>> collect = reducebykeyRDD.collect();
        for(Tuple2 tup:collect){
            System.out.println(tup);
        }
        Integer num = accumulator.value();
        System.out.println("一共有："+num+"行");
        sc.close();
    }
}

结果输出：

// :: INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID ) in  ms on localhost (executor driver) (/)
// :: INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID ) in  ms on localhost (executor driver) (/)
// :: INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool
// :: INFO DAGScheduler: ResultStage  (collect at totalization_device.java:) finished in 0.051 s
// :: INFO DAGScheduler: Job  finished: collect at totalization_device.java:, took 0.273877 s
(,)
(authentication,)
(Registered,)
(is,)
(Found,)
(master.Master:,)
(spark.SecurityManager:,)
(util.log:,)
(,)
(modify,)
(classes,)
(,)
([jar:file:/opt/workspace/hive-3.1./lib/log4j-slf4j-impl-2.10..jar!/org/slf4j/impl/StaticLoggerBinder.class],)
(.,)
(type,)
(with,)
(INFO,)
(permissions:,)
(groups,)
(using,)
(//,)
(Class,)
(@1326ms,)
(WARN,)
(root,)
(signal,)
('MasterUI',)
(,)
(,)
(Set(root);,)
(version,)
(,)
(ui,)
(,)
(load,)
(Set();,)
(,)
(,)
(::,)
(Actual,)
(initialized,)
(server.Server:,)
(master,)
(,)
(multiple,)
(56130C,)
(handler,)
(,)
(,)
(TERM,)
(,)
(daemon,)
(bindings.,)
(builtin-java,)
(server.AbstractConnector:,)
(users,)
([jar:file:/opt/workspace/hbase-1.4./lib/slf4j-log4j12-1.7..jar!/org/slf4j/impl/StaticLoggerBinder.class],)
(http://www.slf4j.org/codes.html#multiple_bindings,1)
(105L,,)
(Starting,)
(jetty-9.3.z-SNAPSHOT,)
(Spark,)
(,)
(SLF4J,)
(platform...,)
(,)
(util.NativeCodeLoader:,)
(Successfully,)
(on,)
('sparkMaster',)
(library,)
(service,)
(,)
(at,)
(in,)
(,)
(@master1,)
(See,)
(.,)
(Logging,)
(missions:,)
(util.Utils:,)
(spark://master1:7077,1)
(for,)
(Changing,)
(,)
(native-hadoop,)
(port,)
(Running,)
(explanation.,)
(your,)
(view,)
(acls,)
(,)
(Unable,)
(binding,)
(to:,)
(disabled;,)
(contains,)
(util.SignalUtils:,)
(process,)
(,)
(SLF4J:,)
(ServerConnector@1cbf22af{HTTP/1.1,[http/1.1]}{0.0.0.0:},)
(,)
(,)
(,)
(SecurityManager:,)
(Started,)
(INT,)
(Set(),)
("spark-root-org.apache.spark.deploy.master.Master-1-master1.out",)
(to,)
(applicable,)
(HUP,)
(started,)
(of,)
(path,)
(where,)
(,)
(an,)
([jar:file:/opt/workspace/hadoop-2.9./share/hadoop/common/lib/slf4j-log4j12-1.7..jar!/org/slf4j/impl/StaticLoggerBinder.class],)
([org.slf4j.impl.Log4jLoggerFactory],)
(2.3.,)
(::,)
(@1280ms,)
(name:,)
(per,)
一共有：25行
// :: INFO SparkUI: Stopped Spark web UI at http://hadoop:4040
// :: INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
// :: INFO MemoryStore: MemoryStore cleared
// :: INFO BlockManager: BlockManager stopped
// :: INFO BlockManagerMaster: BlockManagerMaster stopped
// :: INFO