这里主要分析FileMessageSet类

这个类主要是管理log消息的内存对象和文件对象的类.源代码文件在log目录下.这个类被LogSegment类代理调用用来管理分片.

下面是完整代码.代码比较简单.就不做过多说明了.这个类是MessageSet抽象类的实现类.

class FileMessageSet private[kafka](@volatile var file: File,
private[log] val channel: FileChannel,
private[log] val start: Int,
private[log] val end: Int,
isSlice: Boolean) extends MessageSet with Logging { /* the size of the message set in bytes */
private val _size =
if(isSlice)
new AtomicInteger(end - start) // don't check the file size if this is just a slice view
else
new AtomicInteger(math.min(channel.size().toInt, end) - start) /* if this is not a slice, update the file pointer to the end of the file */
if (!isSlice)
/* set the file position to the last byte in the file */
channel.position(channel.size) /**
* Create a file message set with no slicing.
*/
def this(file: File, channel: FileChannel) =
this(file, channel, start = 0, end = Int.MaxValue, isSlice = false) /**
* Create a file message set with no slicing
*/
def this(file: File) =
this(file, Utils.openChannel(file, mutable = true)) /**
* Create a file message set with mutable option
*/
def this(file: File, mutable: Boolean) = this(file, Utils.openChannel(file, mutable)) /**
* Create a slice view of the file message set that begins and ends at the given byte offsets
*/
def this(file: File, channel: FileChannel, start: Int, end: Int) =
this(file, channel, start, end, isSlice = true) /**
* Return a message set which is a view into this set starting from the given position and with the given size limit.
*
* If the size is beyond the end of the file, the end will be based on the size of the file at the time of the read.
*
* If this message set is already sliced, the position will be taken relative to that slicing.
*
* @param position The start position to begin the read from
* @param size The number of bytes after the start position to include
*
* @return A sliced wrapper on this message set limited based on the given position and size
*/
def read(position: Int, size: Int): FileMessageSet = { //返回读取段对象
if(position < 0)
throw new IllegalArgumentException("Invalid position: " + position)
if(size < 0)
throw new IllegalArgumentException("Invalid size: " + size)
new FileMessageSet(file,
channel,
start = this.start + position,
end = math.min(this.start + position + size, sizeInBytes()))
} /**
* Search forward for the file position of the last offset that is greater than or equal to the target offset
* and return its physical position. If no such offsets are found, return null.
* @param targetOffset The offset to search for.
* @param startingPosition The starting position in the file to begin searching from.
*/
def searchFor(targetOffset: Long, startingPosition: Int): OffsetPosition = { //搜索读写点的方法
var position = startingPosition
val buffer = ByteBuffer.allocate(MessageSet.LogOverhead)
val size = sizeInBytes()
while(position + MessageSet.LogOverhead < size) {
buffer.rewind()
channel.read(buffer, position)
if(buffer.hasRemaining)
throw new IllegalStateException("Failed to read complete buffer for targetOffset %d startPosition %d in %s"
.format(targetOffset, startingPosition, file.getAbsolutePath))
buffer.rewind()
val offset = buffer.getLong()
if(offset >= targetOffset)
return OffsetPosition(offset, position)
val messageSize = buffer.getInt()
if(messageSize < Message.MessageOverhead)
throw new IllegalStateException("Invalid message size: " + messageSize)
position += MessageSet.LogOverhead + messageSize
}
null
} /**
* Write some of this set to the given channel.
* @param destChannel The channel to write to.
* @param writePosition The position in the message set to begin writing from.
* @param size The maximum number of bytes to write
* @return The number of bytes actually written.
*/
def writeTo(destChannel: GatheringByteChannel, writePosition: Long, size: Int): Int = { //主要写方法
// Ensure that the underlying size has not changed.
val newSize = math.min(channel.size().toInt, end) - start
if (newSize < _size.get()) {
throw new KafkaException("Size of FileMessageSet %s has been truncated during write: old size %d, new size %d"
.format(file.getAbsolutePath, _size.get(), newSize))
}
val bytesTransferred = channel.transferTo(start + writePosition, math.min(size, sizeInBytes), destChannel).toInt
trace("FileMessageSet " + file.getAbsolutePath + " : bytes transferred : " + bytesTransferred
+ " bytes requested for transfer : " + math.min(size, sizeInBytes))
bytesTransferred
} /**
* Get a shallow iterator over the messages in the set.
*/
override def iterator() = iterator(Int.MaxValue) /**
* Get an iterator over the messages in the set. We only do shallow iteration here.
* @param maxMessageSize A limit on allowable message size to avoid allocating unbounded memory.
* If we encounter a message larger than this we throw an InvalidMessageException.
* @return The iterator.
*/
def iterator(maxMessageSize: Int): Iterator[MessageAndOffset] = { //主要构造器.
new IteratorTemplate[MessageAndOffset] {
var location = start
val sizeOffsetBuffer = ByteBuffer.allocate(12) override def makeNext(): MessageAndOffset = {
if(location >= end)
return allDone() // read the size of the item
sizeOffsetBuffer.rewind()
channel.read(sizeOffsetBuffer, location)
if(sizeOffsetBuffer.hasRemaining)
return allDone() sizeOffsetBuffer.rewind()
val offset = sizeOffsetBuffer.getLong()
val size = sizeOffsetBuffer.getInt()
if(size < Message.MinHeaderSize)
return allDone()
if(size > maxMessageSize)
throw new InvalidMessageException("Message size exceeds the largest allowable message size (%d).".format(maxMessageSize)) // read the item itself
val buffer = ByteBuffer.allocate(size)
channel.read(buffer, location + 12)
if(buffer.hasRemaining)
return allDone()
buffer.rewind() // increment the location and return the item
location += size + 12
new MessageAndOffset(new Message(buffer), offset) //在这里做映射.同ByteBufferMessageSet里的实现方法类似.
}
}
} /**
* The number of bytes taken up by this file set
*/
def sizeInBytes(): Int = _size.get() /**
* Append these messages to the message set
*/
def append(messages: ByteBufferMessageSet) { //追加message的方法.被上层的append方法调用.
val written = messages.writeTo(channel, 0, messages.sizeInBytes)
_size.getAndAdd(written)
} /**
* Commit all written data to the physical disk
*/
def flush() = { //上层刷新方法的最终实现.
channel.force(true)
} /**
* Close this message set
*/
def close() {
flush()
channel.close()
} /**
* Delete this message set from the filesystem
* @return True iff this message set was deleted.
*/
def delete(): Boolean = { //上层delete函数的最终实现方法
Utils.swallow(channel.close()) //关闭内存数据
file.delete() //删除文件
} /**
* Truncate this file message set to the given size in bytes. Note that this API does no checking that the
* given size falls on a valid message boundary.
* @param targetSize The size to truncate to.
* @return The number of bytes truncated off
*/
def truncateTo(targetSize: Int): Int = {
val originalSize = sizeInBytes
if(targetSize > originalSize || targetSize < 0)
throw new KafkaException("Attempt to truncate log segment to " + targetSize + " bytes failed, " +
" size of this log segment is " + originalSize + " bytes.")
channel.truncate(targetSize)
channel.position(targetSize)
_size.set(targetSize)
originalSize - targetSize
} /**
* Read from the underlying file into the buffer starting at the given position
*/
def readInto(buffer: ByteBuffer, relativePosition: Int): ByteBuffer = {
channel.read(buffer, relativePosition + this.start)
buffer.flip()
buffer
} /**
* Rename the file that backs this message set
* @return true iff the rename was successful
*/
def renameTo(f: File): Boolean = {
val success = this.file.renameTo(f)
this.file = f
success
} } object LogFlushStats extends KafkaMetricsGroup {
val logFlushTimer = new KafkaTimer(newTimer("LogFlushRateAndTimeMs", TimeUnit.MILLISECONDS, TimeUnit.SECONDS))
}

Kafka 源代码分析之FileMessageSet的更多相关文章

  1. Kafka 源代码分析之LogManager

    这里分析kafka 0.8.2的LogManager logmanager是kafka用来管理log文件的子系统.源代码文件在log目录下. 这里会逐步分析logmanager的源代码.首先看clas ...

  2. Kafka 源代码分析.

    这里记录kafka源代码笔记.(代码版本是0.8.2.1) kafka的源代码如何下载.这里简单说一下. git clone https://git-wip-us.apache.org/repos/a ...

  3. Kafka 源代码分析之LogSegment

    这里分析kafka LogSegment源代码 通过一步步分析LogManager,Log源代码之后就会发现,最终的log操作都在LogSegment上实现.LogSegment负责分片的读写恢复刷新 ...

  4. kafka 源代码分析之Message(v0.10)

    这里主要更新一下kafka 0.10.0版本的message消息格式的变化. message 的格式在0.10.0的版本里发生了一些变化(相对于0.8.2.1的版本)这里把0.10.0的message ...

  5. Kafka 源代码分析之ByteBufferMessageSet

    这里分析一下message的封装类ByteBufferMessageSet类 ByteBufferMessageSet类的源代码在源代码目录message目录下.这个类主要封装了message,mes ...

  6. Kafka 源代码分析之Log

    这里分析Log对象本身的源代码. Log类是一个topic分区的基础类.一个topic分区的所有基本管理动作.都在这个对象里完成.类源代码文件为Log.scala.在源代码log目录下. Log类是L ...

  7. Kafka 源代码分析之Message

    这里主要分析一下message的格式. 一条message的构成由以下部分组成 val CrcOffset = 0 //crc校验部分和字长 val CrcLength = 4 val MagicOf ...

  8. Kafka 源代码分析之MessageSet

    这里分析MessageSet类 MessageSet是一个抽象类,定义了一条log的一些接口和常量,FileMessageSet就是MessageSet类的实现类.一条日志中存储的log完整格式如下 ...

  9. Kafka 源代码分析之log框架介绍

    这里主要介绍log管理,读写相关的类的调用关系的介绍. 在围绕log的实际处理上.有很多层的封装和调用.这里主要介绍一下调用结构和顺序. 首先从LogManager开始. 调用关系简单如下:LogMa ...

随机推荐

  1. Commonjs规范及Node模块实现

    前面的话 Node在实现中并非完全按照CommonJS规范实现,而是对模块规范进行了一定的取舍,同时也增加了少许自身需要的特性.本文将详细介绍NodeJS的模块实现 引入 nodejs是区别于java ...

  2. Oracle 12C 新特性之非分区表转分区表online clause(不停业务+索引有效)

    12c以前非分区表需要转换为分区, 如果不停业务的话可以使用在线重定义,只有在表进行切换的时候会有短暂的锁表. 12c 中alter table online clause 实现了表上现有的索引有效, ...

  3. 利用有限自动机(finite automata)进行模式匹配

    一.有限自动机定义及基本术语: 一个有限自动机 M 是一个5元组(Q, ,A, Σ, δ),其中: Q 是所有状态的有限集合;  ∈ Q (属于)是初始状态; A ⊆ Q (子集)是接受状态的集合; ...

  4. Java中的局部变量表及使用jclasslib进行查看

    直接上下载地址 jclasslib是一个独立的工具,不是包含在JDK中的工具,需要自己进行下载,下载地址如下: http://downfile.downcc.com/down/JClassLib_wi ...

  5. 【转】典型的JavaScript面试题

    问题1: 作用域(Scope) (function() { "use strict"; var a = b = 5; })(); console.log(b); 控制台(conso ...

  6. PHP中的运算符

    一.算术运算符   运算符 名称 描述 实例 结果 x + y 加 x 和 y 的和 2 + 2 4 x - y 减 x 和 y 的差 5 - 2 3 x * y 乘 x 和 y 的积 5 * 2 1 ...

  7. Reverse Integer 2015年6月23日

    题目: Reverse digits of an integer. Example1: x = , return Example2: x = -, return - 思路:递归 解答: / test ...

  8. 关于win10和sqlserver的兼容性

    本人主要是用.NET开发一些MIS(信息管理系统)系统,如酒店管理系统,医院管理系统,以及其附属的如餐饮管理系统,洗浴管理系统,以及医保管理系统,合疗管理系统,前期开发的产品主要是VS2008+sql ...

  9. C语言socket编程——linux环境

    先写一个服务器端的监听程序,功能室从客户端读取字符,接收到后告知客户端“I got your message: ”+收到的消息:server.c #include <stdio.h> #i ...

  10. Spring Boot 学习(2)

    文 by / 林本托 Tips 做一个终身学习的人. 源代码:github下的/code01/ch2. 配置 Web 应用程序 在上一章中,我们学习了如何创建一个基本的应用程序模板,并添加了一些基本功 ...