Spark分析之MemoryStore

private case class MemoryEntry(value: Any, size: Long, deserialized: Boolean)

class MemoryStore(blockManager: BlockManager, maxMemory: Long)extends BlockStore(blockManager) {
    private val entries = new LinkedHashMap[BlockId, MemoryEntry](32, 0.75f, true)

    //存
    private def tryToPut(blockId: BlockId, value: Any,size: Long,deserialized: Boolean): ResultWithDroppedBlocks = {
         if (enoughFreeSpace) { //空闲内存是否足以容纳block
            val entry = new MemoryEntry(value, size, deserialized)
            entries.synchronized {
                entries.put(blockId, entry) //将Block放置到内部维护的HashMap中
            }
            //如果是反序列话的就以对象数组方式处理，否则就是以字节数组方式处理
            val valuesOrBytes = if (deserialized) "values" else "bytes"
         }else{//告诉BlockManager内存不足以存下该block，是否将其drop到硬盘中(如果该Block允许Disk存储)
            val droppedBlockStatus = blockManager.dropFromMemory(blockId, data)
         }
    }

    //取：直接从HashMap中根据blockid获取即可
    override def getValues(blockId: BlockId): Option[Iterator[Any]] = {
        val entry = entries.synchronized {
            entries.get(blockId)
        }
        if (entry == null) {
            None
        } else if (entry.deserialized) { //反序列话的就以对象数组方式处理
            Some(entry.value.asInstanceOf[Array[Any]].iterator)
        } else { //序列话的就以字节数组方式处理
            val buffer = entry.value.asInstanceOf[ByteBuffer].duplicate() // Doesn't actually copy data
            Some(blockManager.dataDeserialize(blockId, buffer))
        }
    }
}

总结：

1）内部维护了一个LinkedHashMap来管理所有的block，以blockid作为key将block存储在LinkedHashMap中；

2）在MemoryStore中存放block(tryToPut)时，首先调用ensureFreeSpace()确保空闲内存是否足以容纳该block：

　　足：将该block直接加入到LinkedHashMap中去；

　　不足：通过BlockManager.dropFromMemory将该block写入到disk中

3）MemoryStore将序列化后的字节数组或者反序列化后的java对象数组的block存取在Memory中。