RDD的分区相关

分区是rdd的一个属性，每个分区是一个迭代器

分区器是决定数据数据如何分区

RDD划分成许多分区分布到集群的节点上，分区的多少涉及对这个RDD进行并行计算的粒度。用户可以获取分区数和设置分区数目，默认分区数为程序分配到的CPU核数。

spark中，RDD计算是以分区为单位的，而且计算函数都是在对迭代器复合，不需要保存每次计算的结果。

scala> val numrdd=sc.makeRDD(1 to 10,3)
numrdd: org.apache.spark.rdd.RDD[Int] = ParallelCollectionRDD[0] at makeRDD at <console>:24

scala> import org.apache.spark.TaskContext
import org.apache.spark.TaskContext

scala> numrdd.foreach(x=>{println(TaskContext.get.partitionId+"|"+x)})
[Stage 0:>                                                          (0 + 0) / 3]2|7
2|8
2|9
2|10
0|1
0|2
0|3
1|4
1|5
1|6
scala> numrdd.foreach(x=>{println(TaskContext.getPartitionId+"|"+x)})
1|4
1|5
1|6
0|1
0|2
0|3
2|7
2|8
2|9
2|10

-----------------------------------------------------------------------

scala> val parRDD=sc.makeRDD(Array((100,"dog"),(100,"cat"),(200,"pear"),(100,"tiger"),(200,"apple"),(100,"lion"),(200,"banana"),(100,"elephent"),(300,"paper"),(300,"pen"),(200,"pig"),(300,"ballpen")))
parRDD: org.apache.spark.rdd.RDD[(Int, String)] = ParallelCollectionRDD[40] at makeRDD at <console>:25

scala> parRDD.partitions.length
res41: Int = 4

scala> parRDD.foreach(x=>{println(x+"|"+TaskContext.get.partitionId)})
(100,elephent)|3
(200,pear)|1
(300,paper)|3
(200,apple)|2
(100,lion)|2
(200,banana)|2
(100,tiger)|1
(100,dog)|0
(100,cat)|0
(300,pen)|4
(200,pig)|4
(300,ballpen)|4

scala> parRDD.foreach(x=>{println(x+"|"+TaskContext.get.stageId)})
(200,apple)|45
(100,lion)|45
(200,banana)|45
(100,dog)|45
(100,cat)|45
(200,pear)|45
(100,elephent)|45
(300,paper)|45
(100,tiger)|45
(300,pen)|45
(200,pig)|45
(300,ballpen)|45

scala> parRDD.foreach(x=>{println(x+"|"+TaskContext.get.taskAttemptId)})
(200,apple)|190
(100,lion)|190
(200,banana)|190
(100,elephent)|191
(300,paper)|191
(200,pear)|189
(100,tiger)|189
(100,dog)|188
(100,cat)|188
(300,pen)|192
(200,pig)|192
(300,ballpen)|192

scala> parRDD.foreach(x=>{println(x+"|"+TaskContext.get.taskMetrics)})
(100,dog)|org.apache.spark.executor.TaskMetrics@339a1fc
(100,elephent)|org.apache.spark.executor.TaskMetrics@2c0eca15
(200,pear)|org.apache.spark.executor.TaskMetrics@3850cb6d
(200,apple)|org.apache.spark.executor.TaskMetrics@38090055
(100,tiger)|org.apache.spark.executor.TaskMetrics@3850cb6d
(100,cat)|org.apache.spark.executor.TaskMetrics@339a1fc
(300,paper)|org.apache.spark.executor.TaskMetrics@2c0eca15
(100,lion)|org.apache.spark.executor.TaskMetrics@38090055
(200,banana)|org.apache.spark.executor.TaskMetrics@38090055
(300,pen)|org.apache.spark.executor.TaskMetrics@125f9f17
(200,pig)|org.apache.spark.executor.TaskMetrics@125f9f17
(300,ballpen)|org.apache.spark.executor.TaskMetrics@125f9f17

//查看每个分区的数据

scala> def partitionValueWthID(id:Int,iter:Iterator[(Int,String)])=({var result=scala.collection.mutable.Map[Int,List[(Int,String)]](); while(iter.hasNext){var partid=id;if(result.contains(partid)){var elems=result(partid);elems::=iter.next;result(partid)=elems; } else result(partid)=List[(Int,String)]{iter.next}};result.toIterator})

partitionValueWthID: (id: Int, iter: Iterator[(Int, String)])Iterator[(Int, List[(Int, String)])]

scala> def partitionValueWthID(id:Int,iter:Iterator[(Int,String)])=

(

{

var result=scala.collection.mutable.Map[Int,List[(Int,String)]]();

while(iter.hasNext){

var partid=id;

if(result.contains(partid))  //如果分区ID的键存在，则调整键的值

{

var elems=result(partid);

elems::=iter.next;

result(partid)=elems;

}

else  //键值不存在，则直接赋值

result(partid)=List[(Int,String)]{iter.next}

};

result.toIterator

}

)

partitionValueWthID: (id: Int, iter: Iterator[(Int, String)])Iterator[(Int, List[(Int, String)])]

scala> def partitionValueWthID(id:Int,iter:Iterator[(Int,String)])=({var result=scala.collection.mutable.Map[Int,List[(Int,String)]](); while(iter.hasNext){var partid=id;var elem=iter.next;if(result.contains(partid)){var elems=result(partid);elems::=elem;result(partid)=elems; } else result(partid)=List[(Int,String)]{elem}};result.toIterator})
partitionValueWthID: (id: Int, iter: Iterator[(Int, String)])Iterator[(Int, List[(Int, String)])]

scala> parRDD.mapPartitionsWithIndex(partitionValueWthID).collect

scala> parRDD.mapPartitionsWithIndex(partitionValueWthID).collect
res45: Array[(Int, List[(Int, String)])] = Array((0,List((100,cat), (100,dog))), (1,List((100,tiger), (200,pear))), (2,List((200,banana), (100,lion), (200,apple))), (3,List((300,paper), (100,elephent))), (4,List((300,ballpen), (200,pig), (300,pen))))

或者

scala> import org.apache.spark.TaskContext
import org.apache.spark.TaskContext

scala> parRDD.map(x=>(TaskContext.getPartitionId,x)).groupByKey().collect
res44: Array[(Int, Iterable[(Int, String)])] = Array((0,CompactBuffer((100,dog), (100,cat))), (1,CompactBuffer((200,pear), (100,tiger))), (2,CompactBuffer((200,apple), (100,lion), (200,banana))), (3,CompactBuffer((100,elephent), (300,paper))), (4,CompactBuffer((300,pen), (200,pig), (300,ballpen))))

-----------------------

自定义分区

scala> val parRDD=sc.makeRDD(Array((100,"dog"),(100,"cat"),(200,"pear"),(100,"tiger"),(200,"apple"),(100,"lion"),(200,"banana"),(100,"elephent"),(300,"paper"),(300,"pen"),(200,"pig"),(300,"ballpen")))
parRDD: org.apache.spark.rdd.RDD[(Int, String)] = ParallelCollectionRDD[70] at makeRDD at <console>:27

scala> def partitionValueWthID(id:Int,iter:Iterator[(Int,String)])=({var result=scala.collection.mutable.Map[Int,List[(Int,String)]](); while(iter.hasNext){var partid=id;var elem=iter.next;if(result.contains(partid)){var elems=result(partid);elems::=elem;result(partid)=elems; } else result(partid)=List[(Int,String)]{elem}};result.toIterator})
partitionValueWthID: (id: Int, iter: Iterator[(Int, String)])Iterator[(Int, List[(Int, String)])]

scala> class MyPartitioner extends org.apache.spark.Partitioner{
     |   override def numPartitions: Int = 2
     |   override def getPartition(key: Any): Int = {
     |     val k = key.toString.toInt
     |     if(k > 100){
     |       return 1
     |     }else{
     |       return 0
     |     }
     |   }
     | }
defined class MyPartitioner

scala> parRDD.partitionBy(new MyPartitioner).mapPartitionsWithIndex(partitionValueWthID).collect
res25: Array[(Int, List[(Int, String)])] = Array((0,List((100,elephent), (100,lion), (100,tiger), (100,cat), (100,dog))), (1,List((300,ballpen), (200,pig), (300,pen), (300,paper), (200,banana), (200,apple), (200,pear))))

------------------------------------------------------

scala> val arr=parRDD.keys.distinct.collect
arr: Array[Int] = Array(100, 300, 200)

scala> class MyPartitioner1(parts:Array[Int]) extends org.apache.spark.Partitioner{
     |   override def numPartitions: Int = parts.length+1
     |   val rules=new scala.collection.mutable.HashMap[Int,Int]()
     |   var i=1
     |   for(x<-parts)
     |    {
     |    rules+=(x->i)
     |    i+=1
     |   }
     |   override def getPartition(key: Any): Int = {
     |     val k = key.toString.toInt
     |     rules.getOrElse(k,0)
     |   }
     | }
defined class MyPartitioner1

class MyPartitioner1(parts:Array[Int]) extends org.apache.spark.Partitioner{
  override def numPartitions: Int = parts.length+1 //定义分区数

//定义分区规则
  val rules=new scala.collection.mutable.HashMap[Int,Int]() 
  var i=1
  for(x<-parts)
   {
   rules+=(x->i)
   i+=1
  }

//根据传输的key来确定该记录写入哪个分区
  override def getPartition(key: Any): Int = {
    val k = key.toString.toInt
    rules.getOrElse(k,0)
  }
}

scala> parRDD.partitionBy(new MyPartitioner1(arr)).mapPartitionsWithIndex(partitionValueWthID).collect
res55: Array[(Int, List[(Int, String)])] = Array((1,List((100,elephent), (100,lion), (100,tiger), (100,cat), (100,dog))), (2,List((300,ballpen), (300,pen), (300,paper))), (3,List((200,pig), (200,banana), (200,apple), (200,pear))))

-----------------------------------------

repartition和partitionBy的区别

repartition 和 partitionBy 都是对数据进行重新分区，默认都是使用 HashPartitioner，区别在于partitionBy 只能用于 PairRdd,当它们同时都用于 PairRdd时,partitionBy更接近我们的预期。repartition 其实使用了一个随机生成的数来当做 Key

scala> val parRDD=sc.makeRDD(Array((100,"dog"),(100,"cat"),(200,"pear"),(100,"tiger"),(200,"apple"),(101,"lion"),(201,"banana"),(101,"elephent"),(300,"paper"),(300,"pen"),(200,"pig"),(300,"ballpen")))
parRDD: org.apache.spark.rdd.RDD[(Int, String)] = ParallelCollectionRDD[8] at makeRDD at <console>:25

scala> def partitionValueWthID(id:Int,iter:Iterator[(Int,String)])=({var result=scala.collection.mutable.Map[Int,List[(Int,String)]](); while(iter.hasNext){var partid=id;var elem=iter.next;if(result.contains(partid)){var elems=result(partid);elems::=elem;result(partid)=elems; } else result(partid)=List[(Int,String)]{elem}};result.toIterator})
partitionValueWthID: (id: Int, iter: Iterator[(Int, String)])Iterator[(Int, List[(Int, String)])]

scala> parRDD.repartition(4).mapPartitionsWithIndex(partitionValueWthID).collect
res3: Array[(Int, List[(Int, String)])] = Array((0,List((200,pig), (101,elephent), (200,apple), (100,cat))), (1,List((300,ballpen), (300,paper), (101,lion), (200,pear))), (3,List((300,pen), (201,banana), (100,tiger), (100,dog))))

scala> parRDD.partitionBy(new HashPartitioner(4)).mapPartitionsWithIndex(partitionValueWthID).collect
res7: Array[(Int, List[(Int, String)])] = Array((0,List((300,ballpen), (200,pig), (300,pen), (300,paper), (200,apple), (100,tiger), (200,pear), (100,cat), (100,dog))), (1,List((101,elephent), (201,banana), (101,lion))))

spark 2.2源码RDD.scala中的定义
  var position = (new Random(index)).nextInt(numPartitions)

----------------------
RDD分区函数（Partitioner)
分区划分对于shuffle类操作很关键，它决定了该操作的父RDD与子RDD之间的依赖关系。宽依赖或者窄依赖。
spark默认提供两种划分器：哈希分区划分器（HashPartitioner)和范围分区划分器(RangePartitioner),且Partitioner只存在于（K,V)类型的RDD中，非（K,V)类型的partitioner值为None。

scala> val parRDD=sc.makeRDD(Array((100,"dog"),(100,"cat"),(200,"pear"),(100,"tiger"),(200,"apple"),(100,"lion"),(200,"banana"),(100,"elephent"),(300,"paper"),(300,"pen"),(200,"pig"),(300,"ballpen")))
parRDD: org.apache.spark.rdd.RDD[(Int, String)] = ParallelCollectionRDD[63] at makeRDD at <console>:25

scala> nums.partitioner
res18: Option[org.apache.spark.Partitioner] = None

scala> val groupRDD=parRDD.groupByKey()
groupRDD: org.apache.spark.rdd.RDD[(Int, Iterable[String])] = ShuffledRDD[62] at groupByKey at <console>:27

scala> groupRDD.partitioner
res24: Option[org.apache.spark.Partitioner] = Some(org.apache.spark.HashPartitioner@4)

scala> val lenRDD=groupRDD.mapValues(x=>{val arr=x.toArray;arr.length})
lenRDD: org.apache.spark.rdd.RDD[(Int, Int)] = MapPartitionsRDD[70] at mapValues at <console>:29

scala> lenRDD.partitioner
res34: Option[org.apache.spark.Partitioner] = Some(org.apache.spark.HashPartitioner@5)

scala> lenRDD.collect
res35: Array[(Int, Int)] = Array((100,5), (300,3), (200,4))