package core.java;

import java.util.Arrays;

import java.util.List;

import org.apache.spark.SparkConf;

import org.apache.spark.api.java.JavaPairRDD;

import org.apache.spark.api.java.JavaSparkContext;

import scala.Tuple2;

public class JGroupByKey {

    public static void main(String[] args) {

        SparkConf conf = new SparkConf();

        conf.setMaster("local");

        conf.setAppName("union");

        System.setProperty("hadoop.home.dir", "C:/hadoop-2.8.2");

        JavaSparkContext sc = new JavaSparkContext(conf);

        List<Tuple2<String, Integer>> list = Arrays.asList(new Tuple2<String, Integer>("cl1", ),

                new Tuple2<String, Integer>("cl2", ),new Tuple2<String, Integer>("cl3", ),

                new Tuple2<String, Integer>("cl1", ),new Tuple2<String, Integer>("cl1", ),

                new Tuple2<String, Integer>("cl3", ),new Tuple2<String, Integer>("cl2", ));

        JavaPairRDD<String, Integer> listRDD = sc.parallelizePairs(list);

        JavaPairRDD<String, Iterable<Integer>> results  = listRDD.groupByKey();

        System.out.println(results.collect());

        sc.close();

    }

}

//SLF4J: Class path contains multiple SLF4J bindings.

//SLF4J: Found binding in [jar:file:/E:/bigdata/spark-1.4.0-bin-hadoop2.6/lib/spark-assembly-1.4.0-hadoop2.6.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]

//SLF4J: Found binding in [jar:file:/E:/bigdata/spark-1.4.0-bin-hadoop2.6/lib/spark-examples-1.4.0-hadoop2.6.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]

//SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.

//SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]

//Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties

//17/12/27 20:23:41 INFO SparkContext: Running Spark version 1.4.0

//17/12/27 20:23:43 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

//17/12/27 20:23:44 INFO SecurityManager: Changing view acls to:

//17/12/27 20:23:44 INFO SecurityManager: Changing modify acls to:

//17/12/27 20:23:44 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(); users with modify permissions: Set()

//17/12/27 20:23:46 INFO Slf4jLogger: Slf4jLogger started

//17/12/27 20:23:47 INFO Remoting: Starting remoting

//17/12/27 20:23:48 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriver@172.18.3.6:58955]

//17/12/27 20:23:48 INFO Utils: Successfully started service 'sparkDriver' on port 58955.

//17/12/27 20:23:48 INFO SparkEnv: Registering MapOutputTracker

//17/12/27 20:23:49 INFO SparkEnv: Registering BlockManagerMaster

//17/12/27 20:23:49 INFO DiskBlockManager: Created local directory at C:\Users\\AppData\Local\Temp\spark-c1db5ccf-8e4b-4ef9-9a7f-c6ec66d46664\blockmgr-a60ebb60-8b7c-433c-b035-eded748b261b

//17/12/27 20:23:49 INFO MemoryStore: MemoryStore started with capacity 467.6 MB

//17/12/27 20:23:49 INFO HttpFileServer: HTTP File server directory is C:\Users\\AppData\Local\Temp\spark-c1db5ccf-8e4b-4ef9-9a7f-c6ec66d46664\httpd-f45db3a6-b75c-46a5-a1d1-3539b1698cd0

//17/12/27 20:23:49 INFO HttpServer: Starting HTTP Server

//17/12/27 20:23:49 INFO Utils: Successfully started service 'HTTP file server' on port 58959.

//17/12/27 20:23:49 INFO SparkEnv: Registering OutputCommitCoordinator

//17/12/27 20:23:50 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.

//17/12/27 20:23:50 INFO Utils: Successfully started service 'SparkUI' on port 4041.

//17/12/27 20:23:50 INFO SparkUI: Started SparkUI at http://172.18.3.6:4041

//17/12/27 20:23:50 INFO Executor: Starting executor ID driver on host localhost

//17/12/27 20:23:51 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58978.

//17/12/27 20:23:51 INFO NettyBlockTransferService: Server created on 58978

//17/12/27 20:23:51 INFO BlockManagerMaster: Trying to register BlockManager

//17/12/27 20:23:51 INFO BlockManagerMasterEndpoint: Registering block manager localhost:58978 with 467.6 MB RAM, BlockManagerId(driver, localhost, 58978)

//17/12/27 20:23:51 INFO BlockManagerMaster: Registered BlockManager

//17/12/27 20:23:57 INFO SparkContext: Starting job: collect at JGroupByKey.java:27

//17/12/27 20:23:57 INFO DAGScheduler: Registering RDD 0 (parallelizePairs at JGroupByKey.java:25)

//17/12/27 20:23:57 INFO DAGScheduler: Got job 0 (collect at JGroupByKey.java:27) with 1 output partitions (allowLocal=false)

//17/12/27 20:23:57 INFO DAGScheduler: Final stage: ResultStage 1(collect at JGroupByKey.java:27)

//17/12/27 20:23:57 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 0)

//17/12/27 20:23:57 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 0)

//17/12/27 20:23:57 INFO DAGScheduler: Submitting ShuffleMapStage 0 (ParallelCollectionRDD[0] at parallelizePairs at JGroupByKey.java:25), which has no missing parents

//17/12/27 20:23:58 INFO MemoryStore: ensureFreeSpace(2832) called with curMem=0, maxMem=490356080

//17/12/27 20:23:58 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 2.8 KB, free 467.6 MB)

//17/12/27 20:23:58 INFO MemoryStore: ensureFreeSpace(1553) called with curMem=2832, maxMem=490356080

//17/12/27 20:23:58 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 1553.0 B, free 467.6 MB)

//17/12/27 20:23:58 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on localhost:58978 (size: 1553.0 B, free: 467.6 MB)

//17/12/27 20:23:58 INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:874

//17/12/27 20:23:58 INFO DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 0 (ParallelCollectionRDD[0] at parallelizePairs at JGroupByKey.java:25)

//17/12/27 20:23:58 INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks

//17/12/27 20:23:58 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 1627 bytes)

//17/12/27 20:23:58 INFO Executor: Running task 0.0 in stage 0.0 (TID 0)

//17/12/27 20:23:58 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 879 bytes result sent to driver

//17/12/27 20:23:58 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 427 ms on localhost (1/1)

//17/12/27 20:23:58 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool

//17/12/27 20:23:58 INFO DAGScheduler: ShuffleMapStage 0 (parallelizePairs at JGroupByKey.java:25) finished in 0.559 s

//17/12/27 20:23:58 INFO DAGScheduler: looking for newly runnable stages

//17/12/27 20:23:58 INFO DAGScheduler: running: Set()

//17/12/27 20:23:58 INFO DAGScheduler: waiting: Set(ResultStage 1)

//17/12/27 20:23:58 INFO DAGScheduler: failed: Set()

//17/12/27 20:23:58 INFO DAGScheduler: Missing parents for ResultStage 1: List()

//17/12/27 20:23:58 INFO DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[2] at groupByKey at JGroupByKey.java:26), which is now runnable

//17/12/27 20:23:58 INFO MemoryStore: ensureFreeSpace(4000) called with curMem=4385, maxMem=490356080

//17/12/27 20:23:58 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 3.9 KB, free 467.6 MB)

//17/12/27 20:23:58 INFO MemoryStore: ensureFreeSpace(2129) called with curMem=8385, maxMem=490356080

//17/12/27 20:23:58 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 467.6 MB)

//17/12/27 20:23:58 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on localhost:58978 (size: 2.1 KB, free: 467.6 MB)

//17/12/27 20:23:58 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:874

//17/12/27 20:23:58 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[2] at groupByKey at JGroupByKey.java:26)

//17/12/27 20:23:58 INFO TaskSchedulerImpl: Adding task set 1.0 with 1 tasks

//17/12/27 20:23:58 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 1165 bytes)

//17/12/27 20:23:58 INFO Executor: Running task 0.0 in stage 1.0 (TID 1)

//17/12/27 20:23:59 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks

//17/12/27 20:23:59 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 57 ms

//17/12/27 20:23:59 INFO Executor: Finished task 0.0 in stage 1.0 (TID 1). 5294 bytes result sent to driver

//17/12/27 20:23:59 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 231 ms on localhost (1/1)

//17/12/27 20:23:59 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool

//17/12/27 20:23:59 INFO DAGScheduler: ResultStage 1 (collect at JGroupByKey.java:27) finished in 0.236 s

//17/12/27 20:23:59 INFO DAGScheduler: Job 0 finished: collect at JGroupByKey.java:27, took 1.963917 s

//[(cl3,[97, 90]), (cl1,[90, 96, 89]), (cl2,[91, 60])]

//17/12/27 20:23:59 INFO SparkUI: Stopped Spark web UI at http://172.18.3.6:4041

//17/12/27 20:23:59 INFO DAGScheduler: Stopping DAGScheduler

//17/12/27 20:23:59 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!

//17/12/27 20:23:59 INFO Utils: path = C:\Users\\AppData\Local\Temp\spark-c1db5ccf-8e4b-4ef9-9a7f-c6ec66d46664\blockmgr-a60ebb60-8b7c-433c-b035-eded748b261b, already present as root for deletion.

//17/12/27 20:23:59 INFO MemoryStore: MemoryStore cleared

//17/12/27 20:23:59 INFO BlockManager: BlockManager stopped

//17/12/27 20:23:59 INFO BlockManagerMaster: BlockManagerMaster stopped

//17/12/27 20:23:59 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!

//17/12/27 20:23:59 INFO SparkContext: Successfully stopped SparkContext

//17/12/27 20:23:59 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.

//17/12/27 20:23:59 INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports.

//17/12/27 20:23:59 INFO Utils: Shutdown hook called

//17/12/27 20:23:59 INFO Utils: Deleting directory C:\Users\\AppData\Local\Temp\spark-c1db5ccf-8e4b-4ef9-9a7f-c6ec66d46664

Spark_JGroupByKey的更多相关文章

随机推荐

  1. wfp（Application的运用）

    1.Application中封装了wpf的一些常用的功能,如检查未经处理的异常,处理命令行参数等. 2.如何启动wpf程序: app.xaml可用于启动wpf的窗体.也可以不用这用方式.直接创建一个a ...

  2. Mac终端的Cocoapods的安装及使用

    阅读目录 第一步,首先要检查Mac是否安装了rvm.打开终端,输入指令 rvm -v 第二步,用rvm安装ruby环境 第三步,检查更新RubyGems(Ruby1.9.1 以后的版本自带RubyGe ...

  3. 游戏引擎架构 && windows 核心编程

    欲想正人,必先正己. 静坐当思己过,闲谈莫论人非. 垂直同步的作用: 为避免画面撕裂,许多渲染引擎会在交换缓冲区之前,等待显示器的垂直区间消隐,即电子枪重归屏幕上角的时间. 高分辨率计时器的时间漂移 ...

  4. SQL Server 2008 R2提示评估期已过

    解决SQL Server 2008 r2提示评估期已过 1.注册表把 HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SQL Server\100\Co ...

  5. 重读《深入理解Java虚拟机》七、探究Java内存模型

    1.计算机物理内存结构(物理内存结构决定了虚拟机内存结构) 由于处理器和主存储设备在运算速度上不上同一个数量级上,因此处理器和主存储器之间不得不增设一层高速缓存,将部分在主存储设备的运算放在高速缓存内 ...

  6. Python库源码学习1：Flask之app.run

    先列出app.run()实现的功能,我们以debug=True的情况下进行分析. 1. web服务器,处理http请求 2. 当代码修改后,重启服务器 那么app.run()是如何实现这两个功能的呢? ...

  7. 原生js可视加载图片、延迟加载、懒加载

    <script> //既视加载//var oBtn = document.getElementsByTagName('button')[0];var aImages = document. ...

  8. Postman教程

    1.Postman的介绍 Postman是一款功能强大的用于发送 HTTP 请求的 Chrome插件 .做web页面开发和测试的人员应该都知道!其主要特点:创建 + 测试:创建和发送任何的HTTP请求 ...

  9. docker基本原理

    写的很不错的文章,作个存档 什么是容器 容器是 种轻量级.可移植的为应用程序提供了隔离的运行空间 .每个容器内都包含一个独享的完整用户环境,并且 个容器内的环境变动不会影响其他容器的运行环境,可以使应 ...

  10. js计算常见操作

    如何实现数字相加 var a = 1, var b = 2, var c = a + b 这样c得出来的解果是12, 使用Number()函数可以解决这个问题,如下 var c = Number(a) ...